Merge #17423: ci: Make ci system read-only on the git work tree

fa7523d3aa ci: Extend docs (MarcoFalke)
fa493ef088 ci: Make ci system read-only on the git work tree (MarcoFalke)
fab1333292 ci: Remove git from required packages on host (MarcoFalke)
fa00393bce ci: Make all filesystem operations inside docker (MarcoFalke)

Pull request description:

  Running the ci completely in a docker, without leaving any traces on the host system is not possible right now because the ccache and depends dir needs to be propagated back and picked up by the host for caching.

  Fixes #17372

ACKs for top commit:
  JeremyRubin:
    tested ACK fa7523d3aa

Tree-SHA512: 4bce1a0f883bcbdb34abf409bdbc80d420c5da2045d2f9c5536ac433f9e5b490f23df084546c8c049f688b487572bbfc4f9c4029e9e672f4d9279739d066ed2e

ci/test/00_setup_env.sh

@@ -44,7 +44,7 @@ export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_ROOT_DIR}
 export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST}
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 export WINEDEBUG=${WINEDEBUG:-fixme-all}
-export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3}
+export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git}
 export GOAL=${GOAL:-install}
 export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets}
 export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH

ci/test/04_install.sh

@@ -33,12 +33,6 @@ fi
 mkdir -p "${BASE_SCRATCH_DIR}"
 mkdir -p "${CCACHE_DIR}"
 
-if [ ! -d ${DIR_QA_ASSETS} ]; then
-  git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS}
-fi
-export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
-
-mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/"
 export ASAN_OPTIONS="detect_stack_use_after_return=1"
 export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan"
 export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_BUILD_DIR}/sanitizer-output/tsan"
@@ -54,7 +48,13 @@ if [ -z "$RUN_CI_ON_HOST" ]; then
   echo "Creating $DOCKER_NAME_TAG container to run in"
   ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG"
 
-  DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$BASE_BUILD_DIR,dst=$BASE_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $BASE_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG)
+  DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \
+                  --mount type=bind,src=$BASE_BUILD_DIR,dst=/ro_base,readonly \
+                  --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \
+                  --mount type=bind,src=$BASE_BUILD_DIR/depends,dst=$BASE_BUILD_DIR/depends \
+                  -w $BASE_BUILD_DIR \
+                  --env-file /tmp/env \
+                  $DOCKER_NAME_TAG)
 
   DOCKER_EXEC () {
     docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*"
@@ -83,6 +83,18 @@ if [ "$TRAVIS_OS_NAME" != "osx" ]; then
   ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES
 fi
 
+if [ ! -d ${DIR_QA_ASSETS} ]; then
+  DOCKER_EXEC git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS}
+fi
+export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
+
+DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/"
+
+if [ -z "$RUN_CI_ON_HOST" ]; then
+  echo "Create $BASE_BUILD_DIR"
+  DOCKER_EXEC rsync -a /ro_base/ $BASE_BUILD_DIR
+fi
+
 if [ "$USE_BUSY_BOX" = "true" ]; then
   echo "Setup to use BusyBox utils"
   DOCKER_EXEC mkdir -p $BASE_SCRATCH_DIR/bins/

ci/test/05_before_script.sh

@@ -13,13 +13,13 @@ else
   DOCKER_EXEC echo \> \$HOME/.bitcoin
 fi
 
-mkdir -p depends/SDKs depends/sdk-sources
+DOCKER_EXEC mkdir -p depends/SDKs depends/sdk-sources
 
 if [ -n "$OSX_SDK" ] && [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
   curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
 fi
 if [ -n "$OSX_SDK" ] && [ -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
-  tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+  DOCKER_EXEC tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
 fi
 if [[ $HOST = *-mingw32 ]]; then
   DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\)

ci/test/06_script_a.sh

@@ -19,7 +19,9 @@ else
 fi
 END_FOLD
 
+# Create folder on host and docker, so that `cd` works
 mkdir -p build
+DOCKER_EXEC mkdir -p build
 
 # Temporarily disable errexit, because Travis macOS fails without error message
 set +o errexit
@@ -27,10 +29,12 @@ cd build || (echo "could not enter build directory"; exit 1)
 set -o errexit
 
 BEGIN_FOLD configure
-DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
+DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false)
 END_FOLD
 
 BEGIN_FOLD distdir
+# Create folder on host and docker, so that `cd` works
+mkdir -p "bitcoin-$HOST"
 DOCKER_EXEC make distdir VERSION=$HOST
 END_FOLD
 
@@ -39,7 +43,7 @@ cd "bitcoin-$HOST" || (echo "could not enter distdir bitcoin-$HOST"; exit 1)
 set -o errexit
 
 BEGIN_FOLD configure
-DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
+DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false)
 END_FOLD
 
 set -o errtrace

ci/test/06_script_b.sh

@@ -48,7 +48,3 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then
   DOCKER_EXEC test/fuzz/test_runner.py -l DEBUG ${DIR_FUZZ_IN}
   END_FOLD
 fi
-
-set +o errexit
-cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1)
-set -o errexit