Merge bitcoin/bitcoin#34679: ci: Download script_assets_test.json for Windows CI

fa7612f253 ci: Download script_assets_test.json for Windows CI (MarcoFalke)
7777a13306 test: Move Fetching-print to download_from_url util (MarcoFalke)
faf96286ce test: move-only download_from_url to stand-alone util file (MarcoFalke)
fa0cc1c5a4 test: [doc] Remove outdated comment (MarcoFalke)

Pull request description:

  Fixes https://github.com/bitcoin/bitcoin/issues/34670 by adding a new `download_script_assets` Python helper and calling it.

ACKs for top commit:
  hebasto:
    re-ACK fa7612f253.
  janb84:
    re ACK fa7612f253
  hodlinator:
    utACK fa7612f253

Tree-SHA512: 73c2cb3a31f231174566fb880b82de92734b1679ef000f8d793d774b7e5f5a7b8c7994a3998ca78821115bdc80c16aada69cf596e92c083cf9b9a95c7cee16ea

.github/ci-windows-cross.py

@@ -10,6 +10,9 @@ import subprocess
 import sys
 from pathlib import Path
 
+sys.path.append(str(Path(__file__).resolve().parent.parent / "test"))
+from download_utils import download_script_assets
+
 
 def run(cmd, **kwargs):
     print("+ " + shlex.join(cmd), flush=True)
@@ -81,6 +84,9 @@ def prepare_tests():
     run(cmd_download_prev_rel)
     run([sys.executable, "-m", "pip", "install", "pyzmq"])
 
+    dest = workspace / "unit_test_data"
+    download_script_assets(dest)
+
 
 def run_functional_tests():
     workspace = Path.cwd()
@@ -117,6 +123,8 @@ def run_functional_tests():
 
 
 def run_unit_tests():
+    workspace = Path.cwd()
+    os.environ["DIR_UNIT_TEST_DATA"] = str(workspace / "unit_test_data")
     # Can't use ctest here like other jobs as we don't have a CMake build tree.
     commands = [
         ["./bin/test_bitcoin-qt.exe"],

.github/ci-windows.py

@@ -10,6 +10,9 @@ import subprocess
 import sys
 from pathlib import Path
 
+sys.path.append(str(Path(__file__).resolve().parent.parent / "test"))
+from download_utils import download_script_assets
+
 
 def run(cmd, **kwargs):
     print("+ " + shlex.join(cmd), flush=True)
@@ -126,10 +129,13 @@ def check_manifests(ci_type):
 
 
 def prepare_tests(ci_type):
+    workspace = Path.cwd()
     if ci_type == "standard":
         run([sys.executable, "-m", "pip", "install", "pyzmq"])
+        dest = workspace / "unit_test_data"
+        download_script_assets(dest)
     elif ci_type == "fuzz":
-        repo_dir = str(Path.cwd() / "qa-assets")
+        repo_dir = str(workspace / "qa-assets")
         clone_cmd = [
             "git",
             "clone",
@@ -143,11 +149,13 @@ def prepare_tests(ci_type):
 
 
 def run_tests(ci_type):
-    build_dir = Path.cwd() / "build"
+    workspace = Path.cwd()
+    build_dir = workspace / "build"
     num_procs = str(os.process_cpu_count())
     release_bin = build_dir / "bin" / "Release"
 
     if ci_type == "standard":
+        os.environ["DIR_UNIT_TEST_DATA"] = str(workspace / "unit_test_data")
         test_envs = {
             "BITCOIN_BIN": "bitcoin.exe",
             "BITCOIND": "bitcoind.exe",
@@ -180,7 +188,7 @@ def run_tests(ci_type):
             "--jobs",
             num_procs,
             "--quiet",
-            f"--tmpdirprefix={Path.cwd()}",
+            f"--tmpdirprefix={workspace}",
             "--combinedlogslen=99999999",
             *shlex.split(os.environ.get("TEST_RUNNER_EXTRA", "").strip()),
         ]
@@ -195,7 +203,7 @@ def run_tests(ci_type):
             num_procs,
             "--loglevel",
             "DEBUG",
-            str(Path.cwd() / "qa-assets" / "fuzz_corpora"),
+            str(workspace / "qa-assets" / "fuzz_corpora"),
         ]
         run(fuzz_cmd)
 

.github/workflows/ci.yml

@@ -428,13 +428,12 @@ jobs:
       - name: Check executable manifests
         run: py -3 .github/ci-windows-cross.py check_manifests
 
+      - name: Prepare Windows test environment
+        run: py -3 .github/ci-windows-cross.py prepare_tests
+
       - name: Run unit tests
         run: py -3 .github/ci-windows-cross.py run_unit_tests
 
-      - name: Prepare Windows test environment
-        run: |
-          py -3 .github/ci-windows-cross.py prepare_tests
-
       - name: Run functional tests
         env:
           TEST_RUNNER_EXTRA: "--timeout-factor=${{ env.TEST_RUNNER_TIMEOUT_FACTOR }} ${{ case(github.event_name == 'pull_request', '', '--extended') }}"

test/download_utils.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/license/mit/.
+
+import sys
+import time
+import urllib.request
+
+
+def download_from_url(url, archive):
+    print(f"Fetching: {url}")
+    last_print_time = time.time()
+
+    def progress_hook(progress_bytes, total_size):
+        nonlocal last_print_time
+        now = time.time()
+        percent = min(100, (progress_bytes * 100) / total_size)
+        bar_length = 40
+        filled_length = int(bar_length * percent / 100)
+        bar = '#' * filled_length + '-' * (bar_length - filled_length)
+        if now - last_print_time >= 1 or percent >= 100:
+            print(f'\rDownloading: [{bar}] {percent:.1f}%', flush=True, end="")
+            last_print_time = now
+
+    with urllib.request.urlopen(url) as response:
+        if response.status != 200:
+            raise RuntimeError(f"HTTP request failed with status code: {response.status}")
+
+        sock_info = response.fp.raw._sock.getpeername()
+        print(f"Connected to {sock_info[0]}")
+
+        total_size = int(response.getheader("Content-Length"))
+        progress_bytes = 0
+
+        with open(archive, 'wb') as file:
+            while True:
+                chunk = response.read(8192)
+                if not chunk:
+                    break
+                file.write(chunk)
+                progress_bytes += len(chunk)
+                progress_hook(progress_bytes, total_size)
+
+        if progress_bytes < total_size:
+            raise RuntimeError(f"Download incomplete: expected {total_size} bytes, got {progress_bytes} bytes")
+
+    print('\n', flush=True, end="") # Flush to avoid error output on the same line.
+
+
+def download_script_assets(script_assets_dir):
+    script_assets_dir.mkdir(parents=True, exist_ok=True)
+    script_assets = script_assets_dir / "script_assets_test.json"
+    url = "https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json"
+    try:
+        download_from_url(url, script_assets)
+    except Exception as e:
+        print(f"\nDownload failed: {e}", file=sys.stderr)
+        print("Retrying download after failure ...", file=sys.stderr)
+        time.sleep(12)
+        try:
+            download_from_url(url, script_assets)
+        except Exception as e2:
+            sys.exit(e2)

test/get_previous_releases.py

@@ -3,10 +3,6 @@
 # Copyright (c) 2018-present The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
-#
-# Download or build previous releases.
-# Needs curl and tar to download a release, or the build dependencies when
-# building a release.
 
 import argparse
 import contextlib
@@ -20,9 +16,11 @@ import shutil
 import subprocess
 import sys
 import time
-import urllib.request
 import zipfile
 
+sys.path.append(str(Path(__file__).resolve().parent))
+from download_utils import download_from_url
+
 TAR = os.getenv('TAR', 'tar')
 
 SHA256_SUMS = {
@@ -106,45 +104,6 @@ def pushd(new_dir) -> None:
         os.chdir(previous_dir)
 
 
-def download_from_url(url, archive):
-    last_print_time = time.time()
-
-    def progress_hook(progress_bytes, total_size):
-        nonlocal last_print_time
-        now = time.time()
-        percent = min(100, (progress_bytes * 100) / total_size)
-        bar_length = 40
-        filled_length = int(bar_length * percent / 100)
-        bar = '#' * filled_length + '-' * (bar_length - filled_length)
-        if now - last_print_time >= 1 or percent >= 100:
-            print(f'\rDownloading: [{bar}] {percent:.1f}%', flush=True, end="")
-            last_print_time = now
-
-    with urllib.request.urlopen(url) as response:
-        if response.status != 200:
-            raise RuntimeError(f"HTTP request failed with status code: {response.status}")
-
-        sock_info = response.fp.raw._sock.getpeername()
-        print(f"Connected to {sock_info[0]}")
-
-        total_size = int(response.getheader("Content-Length"))
-        progress_bytes = 0
-
-        with open(archive, 'wb') as file:
-            while True:
-                chunk = response.read(8192)
-                if not chunk:
-                    break
-                file.write(chunk)
-                progress_bytes += len(chunk)
-                progress_hook(progress_bytes, total_size)
-
-        if progress_bytes < total_size:
-            raise RuntimeError(f"Download incomplete: expected {total_size} bytes, got {progress_bytes} bytes")
-
-    print('\n', flush=True, end="") # Flush to avoid error output on the same line.
-
-
 def download_binary(tag, args) -> int:
     if Path(tag).is_dir():
         if not args.remove_dir:
@@ -170,8 +129,6 @@ def download_binary(tag, args) -> int:
     archive = f'bitcoin-{tag[1:]}-{host}.{archive_format}'
     archive_url = f'https://bitcoincore.org/{bin_path}/{archive}'
 
-    print(f'Fetching: {archive_url}')
-
     try:
         download_from_url(archive_url, archive)
     except Exception as e: