Merge bitcoin/bitcoin#32009: contrib: turn off compression of macOS SDK to fix determinism (across distros)

3e01b5d0e7 contrib: rename gen-sdk to gen-sdk.py (fanquake)
c1213a35ab macdeploy: disable compression in macOS gen-sdk script (fanquake)
a33d034545 contrib: more selectively pick files for macOS SDK (fanquake)

Pull request description:

  This includes three changes. The first is to more selectively pick files for inclusion into our macOS SDK tarball (skip manpages, binaries etc), which is nice because it redues the size of the tarball (from ~80mb to 20mb), and makes the size increase that happens with the next commit, less-bad.

  The second change removes compression of the tarball. Starting with Python 3.11, Pythons gzip might delegate to zlib. Depending on the OS, i.e Ubuntu vs Fedora, the underlying zlib implementation might differ, resulting in different output.

  For now, or until a better solution exists, remove compression. This results in the SDK increasing in size to ~157mb. Which is not unreasonable, to regain determinism (and would be significantly worse without the previous commit).

  See: https://docs.python.org/3/library/gzip.html#gzip.compress

  The third renames `gen-sdk` to `gen-sdk.py`, so that it will be linted, along with the rest of our Python files.

  Fixes #31873. We could probably also put this into 30.x.

ACKs for top commit:
  stickies-v:
    ACK 3e01b5d0e7 modulo the new .tar SDK being uploaded
  davidgumberg:
    Tested ACK 3e01b5d0e7

Tree-SHA512: 272164a98e0e6f10822870162c1b3a405693c2f64d3ed085a2d2243a48641d940704b5ef6022256915ac9cf383e87a4f8d4dc2ec4eaa9d25e2abd30f5498778b

ci/test/01_base_install.sh

@@ -90,7 +90,7 @@ mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources"
 OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers"
 
 if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then
-  OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz"
+  OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar"
   OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}"
   if [ ! -f "$OSX_SDK_PATH" ]; then
     ${CI_RETRY_EXE} curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH"

contrib/guix/README.md

@@ -37,7 +37,7 @@ You can then either point to the SDK using the `SDK_PATH` environment variable:
 
 ```sh
 # Extract the SDK tarball to /path/to/parent/dir/of/extracted/SDK/Xcode-<foo>-<bar>-extracted-SDK-with-libcxx-headers
-tar -C /path/to/parent/dir/of/extracted/SDK -xaf /path/to/Xcode-<foo>-<bar>-extracted-SDK-with-libcxx-headers.tar.gz
+tar -C /path/to/parent/dir/of/extracted/SDK -xaf /path/to/Xcode-<foo>-<bar>-extracted-SDK-with-libcxx-headers.tar
 
 # Indicate where to locate the SDK tarball
 export SDK_PATH=/path/to/parent/dir/of/extracted/SDK

contrib/macdeploy/README.md

@@ -44,15 +44,15 @@ xip -x Xcode_15.xip
 
 ### Step 2: Generating the SDK tarball from `Xcode.app`
 
-To generate the SDK, run the script [`gen-sdk`](./gen-sdk) with the
+To generate the SDK, run the script [`gen-sdk.py`](./gen-sdk.py) with the
 path to `Xcode.app` (extracted in the previous stage) as the first argument.
 
 ```bash
-./contrib/macdeploy/gen-sdk '/path/to/Xcode.app'
+./contrib/macdeploy/gen-sdk.py '/path/to/Xcode.app'
 ```
 
-The generated archive should be: `Xcode-15.0-15A240d-extracted-SDK-with-libcxx-headers.tar.gz`.
+The generated archive should be: `Xcode-15.0-15A240d-extracted-SDK-with-libcxx-headers.tar`.
-The `sha256sum` should be `c0c2e7bb92c1fee0c4e9f3a485e4530786732d6c6dd9e9f418c282aa6892f55d`.
+The `sha256sum` should be `95b00dc41fa090747dc0a7907a5031a2fcb2d7f95c9584ba6bccdb99b6e3d498`.
 
 ## Deterministic macOS App Notes
 

contrib/macdeploy/gen-sdk.py

@@ -2,9 +2,7 @@
 import argparse
 import plistlib
 import pathlib
-import sys
 import tarfile
-import gzip
 import os
 import contextlib
 
@@ -22,12 +20,12 @@ def run():
     parser = argparse.ArgumentParser(
         description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
 
-    parser.add_argument('xcode_app', metavar='XCODEAPP', nargs=1)
+    parser.add_argument('xcode_app', metavar='XCODEAPP', type=pathlib.Path)
-    parser.add_argument("-o", metavar='OUTSDKTGZ', nargs=1, dest='out_sdktgz', required=False)
+    parser.add_argument("-o", metavar='OUTSDKTAR', dest='out_sdkt', type=pathlib.Path, required=False)
 
     args = parser.parse_args()
 
-    xcode_app = pathlib.Path(args.xcode_app[0]).resolve()
+    xcode_app = args.xcode_app.resolve()
     assert xcode_app.is_dir(), "The supplied Xcode.app path '{}' either does not exist or is not a directory".format(xcode_app)
 
     xcode_app_plist = xcode_app.joinpath("Contents/version.plist")
@@ -47,11 +45,7 @@ def run():
 
     out_name = "Xcode-{xcode_version}-{xcode_build_id}-extracted-SDK-with-libcxx-headers".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id)
 
-    if args.out_sdktgz:
+    out_sdkt_path = args.out_sdkt or pathlib.Path("./{}.tar".format(out_name))
-        out_sdktgz_path = pathlib.Path(args.out_sdktgz_path)
-    else:
-        # Construct our own out_sdktgz if not specified on the command line
-        out_sdktgz_path = pathlib.Path("./{}.tar.gz".format(out_name))
 
     def tarfp_add_with_base_change(tarfp, dir_to_add, alt_base_dir):
         """Add all files in dir_to_add to tarfp, but prepend alt_base_dir to the files'
@@ -68,6 +62,8 @@ def run():
 
         """
         def change_tarinfo_base(tarinfo):
+            if tarinfo.name and tarinfo.name.endswith((".swiftmodule", ".modulemap")):
+                return None
             if tarinfo.name and tarinfo.name.startswith("./"):
                 tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name))
             if tarinfo.linkname and tarinfo.linkname.startswith("./"):
@@ -81,16 +77,17 @@ def run():
             return tarinfo
         with cd(dir_to_add):
             # recursion already adds entries in sorted order
-            tarfp.add(".", recursive=True, filter=change_tarinfo_base)
+            tarfp.add("./usr/include", recursive=True, filter=change_tarinfo_base)
+            tarfp.add("./usr/lib", recursive=True, filter=change_tarinfo_base)
+            tarfp.add("./System/Library/Frameworks", recursive=True, filter=change_tarinfo_base)
 
-    print("Creating output .tar.gz file...")
+    print("Creating output .tar file...")
-    with out_sdktgz_path.open("wb") as fp:
+    with out_sdkt_path.open("wb") as fp:
-        with gzip.GzipFile(fileobj=fp, mode='wb', compresslevel=9, mtime=0) as gzf:
+        with tarfile.open(mode="w", fileobj=fp, format=tarfile.PAX_FORMAT) as tarfp:
-            with tarfile.open(mode="w", fileobj=gzf, format=tarfile.GNU_FORMAT) as tarfp:
+            print("Adding MacOSX SDK {} files...".format(sdk_version))
-                print("Adding MacOSX SDK {} files...".format(sdk_version))
+            tarfp_add_with_base_change(tarfp, sdk_dir, out_name)
-                tarfp_add_with_base_change(tarfp, sdk_dir, out_name)
+    print("Done! Find the resulting tarball at:")
-    print("Done! Find the resulting gzipped tarball at:")
+    print(out_sdkt_path.resolve())
-    print(out_sdktgz_path.resolve())
 
 if __name__ == '__main__':
     run()