diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index abeed5120..c9a053885 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,6 +23,7 @@ env: # /.travis.yml # /Dockerfile # /dev.Dockerfile + # /make/builder.Dockerfile # /.github/workflows/release.yml GO_VERSION: 1.15.6 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c1aed5196..e40deccd6 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -10,13 +10,11 @@ defaults: shell: bash env: - DOCKER_REPO: lightninglabs - DOCKER_IMAGE: lnd - # If you change this value, please change it in the following files as well: # /.travis.yml # /Dockerfile # /dev.Dockerfile + # /make/builder.Dockerfile # /.github/workflows/main.yml GO_VERSION: 1.15.6 @@ -27,6 +25,8 @@ jobs: steps: - name: git checkout uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: go cache uses: actions/cache@v1 diff --git a/.travis.yml b/.travis.yml index cd1c1cb72..eb4243f6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ go: # If you change this value, please change it in the following files as well: # /Dockerfile # /dev.Dockerfile + # /make/builder.Dockerfile # /.github/workflows/main.yml # /.github/workflows/release.yml - "1.15.6" diff --git a/Dockerfile b/Dockerfile index b544c0b84..9fdcb4a3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,9 @@ +# If you change this value, please change it in the following files as well: +# /.travis.yml +# /dev.Dockerfile +# /make/builder.Dockerfile +# /.github/workflows/main.yml +# /.github/workflows/release.yml FROM golang:1.15.6-alpine as builder # Force Go to use the cgo based DNS resolver. This is required to ensure DNS diff --git a/Makefile b/Makefile index 4b10a2ffa..cfe1de345 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ IOS_BUILD := $(IOS_BUILD_DIR)/Lndmobile.framework ANDROID_BUILD_DIR := $(MOBILE_BUILD_DIR)/android ANDROID_BUILD := $(ANDROID_BUILD_DIR)/Lndmobile.aar -COMMIT := $(shell git describe --abbrev=40 --dirty) +COMMIT := $(shell git describe --tags --dirty) COMMIT_HASH := $(shell git rev-parse HEAD) BTCD_COMMIT := $(shell cat go.mod | \ @@ -156,6 +156,14 @@ release: $(VERSION_CHECK) ./scripts/release.sh build-release "$(VERSION_TAG)" "$(BUILD_SYSTEM)" "$(RELEASE_TAGS)" "$(RELEASE_LDFLAGS)" +docker-release: + @$(call print, "Building release helper docker image.") + if [ "$(tag)" = "" ]; then echo "Must specify tag=!"; exit 1; fi + + docker build -t lnd-release-helper -f make/builder.Dockerfile make/ + $(DOCKER_RELEASE_HELPER) scripts/release.sh check-tag "$(VERSION_TAG)" + $(DOCKER_RELEASE_HELPER) scripts/release.sh build-release "$(VERSION_TAG)" "$(BUILD_SYSTEM)" "$(RELEASE_TAGS)" "$(RELEASE_LDFLAGS)" + scratch: build diff --git a/dev.Dockerfile b/dev.Dockerfile index cd3be28ae..f876febca 100644 --- a/dev.Dockerfile +++ b/dev.Dockerfile @@ -1,3 +1,9 @@ +# If you change this value, please change it in the following files as well: +# /.travis.yml +# /Dockerfile +# /make/builder.Dockerfile +# /.github/workflows/main.yml +# /.github/workflows/release.yml FROM golang:1.15.6-alpine as builder LABEL maintainer="Olaoluwa Osuntokun " diff --git a/docs/release.md b/docs/release.md index e6591ea47..58d82d845 100644 --- a/docs/release.md +++ b/docs/release.md @@ -10,7 +10,23 @@ utilize a work around needed until `go1.13.2`. ## Building a New Release -### macOS/Linux/Windows (WSL) +### MacOS + +The first requirement is to have [`docker`](https://www.docker.com/) +installed locally and running. The second requirement is to have `make` +installed. Everything else (including `golang`) is included in the release +helper image. + +To build a release, run the following commands: + +1. `git clone https://github.com/lightningnetwork/lnd.git` +2. `cd lnd` +3. `git checkout # is the name of the next release/tag` +4. `make docker-release tag=` + +Where `` is the name of the next release of `lnd`. + +### Linux/Windows (WSL) No prior set up is needed on Linux or macOS is required in order to build the release binaries. However, on Windows, the only way to build the release @@ -19,7 +35,8 @@ the release binaries following these steps: 1. `git clone https://github.com/lightningnetwork/lnd.git` 2. `cd lnd` -3. `make release tag= # is the name of the next release/tag` +3. `git checkout # is the name of the next release/tag` +4. `make release tag=` This will then create a directory of the form `lnd-` containing archives of the release binaries for each supported operating system and architecture, diff --git a/make/builder.Dockerfile b/make/builder.Dockerfile new file mode 100644 index 000000000..08ba05c60 --- /dev/null +++ b/make/builder.Dockerfile @@ -0,0 +1,34 @@ +# If you change this value, please change it in the following files as well: +# /.travis.yml +# /Dockerfile +# /dev.Dockerfile +# /.github/workflows/main.yml +# /.github/workflows/release.yml +FROM golang:1.15.6-buster + +MAINTAINER Olaoluwa Osuntokun + +# Golang build related environment variables that are static and used for all +# architectures/OSes. +ENV GODEBUG netdns=cgo +ENV GO111MODULE=auto +ENV CGO_ENABLED=0 + +# Set up cache directories. Those will be mounted from the host system to speed +# up builds. If go isn't installed on the host system, those will fall back to +# temp directories during the build (see make/release_flags.mk). +ENV GOCACHE=/tmp/build/.cache +ENV GOMODCACHE=/tmp/build/.modcache + +RUN apt-get update && apt-get install -y \ + git \ + make \ + tar \ + zip \ + bash \ + && mkdir -p /tmp/build/lnd \ + && mkdir -p /tmp/build/.cache \ + && mkdir -p /tmp/build/.modcache \ + && chmod -R 777 /tmp/build/ + +WORKDIR /tmp/build/lnd diff --git a/make/release_flags.mk b/make/release_flags.mk index e0058133e..8a00f0898 100644 --- a/make/release_flags.mk +++ b/make/release_flags.mk @@ -1,6 +1,16 @@ VERSION_TAG = $(shell date +%Y%m%d)-01 VERSION_CHECK = @$(call print, "Building master with date version tag") +DOCKER_RELEASE_HELPER = docker run \ + -it \ + --rm \ + --user $(shell id -u):$(shell id -g) \ + -v $(shell pwd):/tmp/build/lnd \ + -v $(shell bash -c "go env GOCACHE || (mkdir -p /tmp/go-cache; echo /tmp/go-cache)"):/tmp/build/.cache \ + -v $(shell bash -c "go env GOMODCACHE || (mkdir -p /tmp/go-modcache; echo /tmp/go-modcache)"):/tmp/build/.modcache \ + -e SKIP_VERSION_CHECK \ + lnd-release-helper + BUILD_SYSTEM = darwin-amd64 \ dragonfly-amd64 \ freebsd-386 \ diff --git a/scripts/release.sh b/scripts/release.sh index e73ab3ba2..4b89bbac4 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -13,6 +13,61 @@ LND_VERSION_REGEX="lnd version (.+) commit" PKG="github.com/lightningnetwork/lnd" PACKAGE=lnd +# Needed for setting file timestamps to get reproducible archives. +BUILD_DATE="2020-01-01 00:00:00" +BUILD_DATE_STAMP="202001010000.00" + +# reproducible_tar_gzip creates a reproducible tar.gz file of a directory. This +# includes setting all file timestamps and ownership settings uniformly. +function reproducible_tar_gzip() { + local dir=$1 + local tar_cmd=tar + + # MacOS has a version of BSD tar which doesn't support setting the --mtime + # flag. We need gnu-tar, or gtar for short to be installed for this script to + # work properly. + tar_version=$(tar --version) + if [[ ! "$tar_version" =~ "GNU tar" ]]; then + if ! command -v "gtar"; then + echo "GNU tar is required but cannot be found!" + echo "On MacOS please run 'brew install gnu-tar' to install gtar." + exit 1 + fi + + # We have gtar installed, use that instead. + tar_cmd=gtar + fi + + # Pin down the timestamp time zone. + export TZ=UTC + + find "${dir}" -print0 | LC_ALL=C sort -r -z | $tar_cmd \ + "--mtime=${BUILD_DATE}" --no-recursion --null --mode=u+rw,go+r-w,a+X \ + --owner=0 --group=0 --numeric-owner -c -T - | gzip -9n > "${dir}.tar.gz" + + rm -r "${dir}" +} + +# reproducible_zip creates a reproducible zip file of a directory. This +# includes setting all file timestamps. +function reproducible_zip() { + local dir=$1 + + # Pin down file name encoding and timestamp time zone. + export TZ=UTC + + # Set the date of each file in the directory that's about to be packaged to + # the same timestamp and make sure the same permissions are used everywhere. + chmod -R 0755 "${dir}" + touch -t "${BUILD_DATE_STAMP}" "${dir}" + find "${dir}" -print0 | LC_ALL=C sort -r -z | xargs -0r touch \ + -t "${BUILD_DATE_STAMP}" + + find "${dir}" | LC_ALL=C sort -r | zip -o -X -r -@ "${dir}.zip" + + rm -r "${dir}" +} + # green prints one line of green text (if the terminal supports it). function green() { echo -e "\e[0;32m${1}\e[0m" @@ -37,7 +92,7 @@ function check_tag_correct() { fi # If a tag is specified, ensure that that tag is present and checked out. - if [[ $tag != $(git describe) ]]; then + if [[ $tag != $(git describe --tags) ]]; then red "tag $tag not checked out" exit 1 fi @@ -82,20 +137,27 @@ function build_release() { green " - Packaging vendor" go mod vendor - tar -czf vendor.tar.gz vendor + reproducible_tar_gzip vendor maindir=$PACKAGE-$tag mkdir -p $maindir + mv vendor.tar.gz "${maindir}/" - cp vendor.tar.gz $maindir/ - rm vendor.tar.gz - rm -r vendor + # Don't use tag in source directory, otherwise our file names get too long and + # tar starts to package them non-deterministically. + package_source="${PACKAGE}-source" - package_source="${maindir}/${PACKAGE}-source-${tag}.tar" - git archive -o "${package_source}" HEAD - gzip -f "${package_source}" >"${package_source}.gz" + # The git archive command doesn't support setting timestamps and file + # permissions. That's why we unpack the tar again, then use our reproducible + # method to create the final archive. + git archive -o "${maindir}/${package_source}.tar" HEAD cd "${maindir}" + mkdir -p ${package_source} + tar -xf "${package_source}.tar" -C ${package_source} + rm "${package_source}.tar" + reproducible_tar_gzip ${package_source} + mv "${package_source}.tar.gz" "${package_source}-$tag.tar.gz" for i in $sys; do os=$(echo $i | cut -f1 -d-) @@ -120,15 +182,13 @@ function build_release() { popd if [[ $os == "windows" ]]; then - zip -r "${dir}.zip" "${dir}" + reproducible_zip "${dir}" else - tar -cvzf "${dir}.tar.gz" "${dir}" + reproducible_tar_gzip "${dir}" fi - - rm -r "${dir}" done - shasum -a 256 * >manifest-$tag.txt + sha256sum * >manifest-$tag.txt } # usage prints the usage of the whole script.