Merge bitcoin/bitcoin#34391: lint: upgrade lint scripts for worktrees

5aeaa71c77 lint: pass args from lint.py to cargo run in container (will)
c17a2adb8d lint: upgrade lint scripts for worktrees (will)

Pull request description:

  Fixes #29972

  Use a single script to run the linter locally or in CI.

  Works from inside a worktree.

ACKs for top commit:
  maflcko:
    review ACK 5aeaa71c77 🔒
  davidgumberg:
    code review and lightly tested reACK 5aeaa71
  l0rinc:
    Tested (+ lightly reviewed) ACK 5aeaa71c77

Tree-SHA512: 7c11f649b4752739d31c4f9e6306a98bd2e615b27a0819bbb5e7d9284b9e28bd9f424e145f16361f672f1a63441a1ae2f901c4f99759e997b72a4bf2d56d8d39

.github/ci-lint-exec.py

@@ -1,55 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or https://opensource.org/license/mit.
-
-import os
-import shlex
-import subprocess
-import sys
-import time
-
-
-def run(cmd, **kwargs):
-    print("+ " + shlex.join(cmd), flush=True)
-    kwargs.setdefault("check", True)
-    try:
-        return subprocess.run(cmd, **kwargs)
-    except Exception as e:
-        sys.exit(e)
-
-
-def main():
-    CONTAINER_NAME = os.environ["CONTAINER_NAME"]
-
-    build_cmd = [
-        "docker", "buildx", "build",
-        f"--tag={CONTAINER_NAME}",
-        *shlex.split(os.getenv("DOCKER_BUILD_CACHE_ARG", "")),
-        "--file=./ci/lint_imagefile",
-        "."
-    ]
-
-    if run(build_cmd, check=False).returncode != 0:
-        print("Retry building image tag after failure")
-        time.sleep(3)
-        run(build_cmd)
-
-    extra_env = []
-    if os.environ["GITHUB_EVENT_NAME"] == "pull_request":
-        extra_env = ["--env", "LINT_CI_IS_PR=1"]
-    if os.environ["GITHUB_EVENT_NAME"] != "pull_request" and os.environ["GITHUB_REPOSITORY"] == "bitcoin/bitcoin":
-        extra_env = ["--env", "LINT_CI_SANITY_CHECK_COMMIT_SIG=1"]
-
-    run([
-        "docker",
-        "run",
-        "--rm",
-        *extra_env,
-        f"--volume={os.getcwd()}:/bitcoin",
-        CONTAINER_NAME,
-    ])
-
-
-if __name__ == "__main__":
-    main()

.github/workflows/ci.yml

@@ -681,4 +681,6 @@ jobs:
           cache-provider: ${{ needs.runners.outputs.provider }}
 
       - name: CI script
-        run: python .github/ci-lint-exec.py
+        run: |
+          git worktree add ../lint-worktree HEAD
+          ../lint-worktree/ci/lint.py

ci/lint.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/license/mit/.
+
+import os
+import shlex
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+
+def run(cmd, **kwargs):
+    print("+ " + shlex.join(cmd), flush=True)
+    kwargs.setdefault("check", True)
+    try:
+        return subprocess.run(cmd, **kwargs)
+    except Exception as e:
+        sys.exit(str(e))
+
+
+def get_worktree_mounts(repo_root):
+    git_path = repo_root / ".git"
+    if not git_path.is_file():
+        return []
+    content = git_path.read_text().strip()
+    if not content.startswith("gitdir: "):
+        return []
+    gitdir = (repo_root / content.removeprefix("gitdir: ")).resolve()
+    main_gitdir = gitdir.parent.parent
+    return [
+        f"--volume={gitdir}:{gitdir}",
+        f"--volume={main_gitdir}:{main_gitdir}:ro",
+    ]
+
+
+def main():
+    repo_root = Path(__file__).resolve().parent.parent
+    is_ci = os.environ.get("GITHUB_ACTIONS") == "true"
+    container = "bitcoin-linter"
+
+    build_cmd = [
+        "docker",
+        "buildx",
+        "build",
+        f"--tag={container}",
+        *shlex.split(os.environ.get("DOCKER_BUILD_CACHE_ARG", "")),
+        f"--file={repo_root}/ci/lint_imagefile",
+        str(repo_root),
+    ]
+    if run(build_cmd, check=False).returncode != 0:
+        if is_ci:
+            print("Retry building image after failure")
+            time.sleep(3)
+        run(build_cmd)
+
+    extra_env = []
+    if is_ci:
+        if os.environ.get("GITHUB_EVENT_NAME") == "pull_request":
+            extra_env = ["--env", "LINT_CI_IS_PR=1"]
+        elif os.environ.get("GITHUB_REPOSITORY") == "bitcoin/bitcoin":
+            extra_env = ["--env", "LINT_CI_SANITY_CHECK_COMMIT_SIG=1"]
+
+    run(
+        [
+            "docker",
+            "run",
+            "--rm",
+            *extra_env,
+            f"--volume={repo_root}:/bitcoin",
+            *get_worktree_mounts(repo_root),
+            *([] if is_ci else ["-it"]),
+            container,
+            *sys.argv[1:],
+        ]
+    )
+
+
+if __name__ == "__main__":
+    main()

ci/lint/06_script.sh

@@ -16,7 +16,7 @@ if [ -n "${LINT_CI_IS_PR}" ]; then
   fi
 fi
 
-RUST_BACKTRACE=1 cargo run --manifest-path "./test/lint/test_runner/Cargo.toml"
+RUST_BACKTRACE=1 cargo run --manifest-path "./test/lint/test_runner/Cargo.toml" -- "$@"
 
 if [ "${LINT_CI_SANITY_CHECK_COMMIT_SIG}" = "1" ] ; then
     # Sanity check only the last few commits to get notified of missing sigs,

ci/lint/container-entrypoint.sh

@@ -12,8 +12,4 @@ git config --global --add safe.directory /bitcoin
 
 export PATH="/python_build/bin:${PATH}"
 
-if [ -z "$1" ]; then
+./ci/lint/06_script.sh "$@"
-  bash -ic "./ci/lint/06_script.sh"
-else
-  exec "$@"
-fi

test/lint/README.md

@@ -3,20 +3,24 @@ This folder contains lint scripts.
 Running locally
 ===============
 
-To run linters locally with the same versions as the CI environment, use the included
+To run linters locally with the same versions as the CI environment use
-Dockerfile:
+the _lint.py_ helper script which runs checks inside the CI container:
 
 ```sh
-DOCKER_BUILDKIT=1 docker build --platform=linux --tag=bitcoin-linter --file="./ci/lint_imagefile" ./ && docker run --rm -v $(pwd):/bitcoin -it bitcoin-linter
+./ci/lint.py
 ```
 
-Building the container can be done every time, because it is fast when the
+Extra arguments are passed to `cargo run -- ...` in the container so you can do:
-result is cached and it prevents issues when the image changes.
+
+```sh
+./ci/lint.py --help
+./ci/lint.py --lint=py_lint
+```
 
 test runner
 ===========
 
-To run all the lint checks in the test runner outside the docker you first need
+To run all the lint checks in the test runner outside the container you first need
 to install the rust toolchain using your package manager of choice or
 [rustup](https://www.rust-lang.org/tools/install).