commit 9cfb954a72d83e25d132940a88177273bcf812cc
parent 16b14d6f06f712aebfe09fdae0e80eb6aaf401e0
Author: ghost43 <somber.night@protonmail.com>
Date: Sat, 13 Jun 2020 01:53:08 +0000
Merge pull request #6228 from SomberNight/202006_reproducible_targz
make targz sdist reproducible, rm zip
Diffstat:
13 files changed, 188 insertions(+), 62 deletions(-)
diff --git a/.travis.yml b/.travis.yml
@@ -99,6 +99,26 @@ jobs:
script:
- sudo docker run --name electrum-appimage-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/appimage electrum-appimage-builder-img ./build.sh
after_success: true
+ - if: branch = master
+ name: "tarball build"
+ language: c
+ python: false
+ services:
+ - docker
+ before_install:
+ # hack: travis already cloned the repo, but we re-clone now, as we need to have umask set BEFORE cloning
+ - umask 0022
+ - mkdir fresh_clone && cd fresh_clone
+ - git clone https://github.com/$TRAVIS_REPO_SLUG.git && cd electrum
+ - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then git fetch origin pull/$TRAVIS_PULL_REQUEST/merge; fi
+ - git checkout $TRAVIS_COMMIT
+ - echo "Second git clone ready at $PWD"
+ install:
+ - sudo docker build --no-cache -t electrum-sdist-builder-img ./contrib/build-linux/sdist/
+ script:
+ - echo "Building sdist at $PWD"
+ - sudo docker run --name electrum-sdist-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/sdist electrum-sdist-builder-img ./build.sh
+ after_success: true
- stage: release check
install:
- git fetch --all --tags
diff --git a/README.rst b/README.rst
@@ -121,7 +121,7 @@ Creating Binaries
Linux (tarball)
---------------
-See :code:`contrib/build-linux/README.md`.
+See :code:`contrib/build-linux/sdist/README.md`.
Linux (AppImage)
diff --git a/contrib/build-linux/README.md b/contrib/build-linux/README.md
@@ -1,16 +0,0 @@
-Source tarballs
-===============
-
-✗ _This script does not produce reproducible output (yet!)._
-
-1. Prepare python dependencies used by Electrum.
-
- ```
- contrib/make_packages
- ```
-
-2. Create source tarball.
-
- ```
- contrib/make_tgz
- ```
diff --git a/contrib/build-linux/appimage/README.md b/contrib/build-linux/appimage/README.md
@@ -61,6 +61,11 @@ diff sha256sum1 sha256sum2 > d
cat d
```
+For file metadata, e.g. timestamps:
+```
+rsync -n -a -i --delete squashfs-root1/ squashfs-root2/
+```
+
Useful binary comparison tools:
- vbindiff
- diffoscope
diff --git a/contrib/build-linux/sdist/Dockerfile b/contrib/build-linux/sdist/Dockerfile
@@ -0,0 +1,17 @@
+FROM ubuntu:20.04@sha256:5747316366b8cc9e3021cd7286f42b2d6d81e3d743e2ab571f55bcd5df788cc8
+
+ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -q && \
+ apt-get install -qy \
+ git \
+ gettext \
+ python3 \
+ python3-pip \
+ python3-setuptools \
+ faketime \
+ && \
+ rm -rf /var/lib/apt/lists/* && \
+ apt-get autoremove -y && \
+ apt-get clean
diff --git a/contrib/build-linux/sdist/README.md b/contrib/build-linux/sdist/README.md
@@ -0,0 +1,52 @@
+Source tarballs
+===============
+
+✓ _This file should be reproducible, meaning you should be able to generate
+ distributables that match the official releases._
+
+This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another
+similar system. The docker commands should be executed in the project's root
+folder.
+
+1. Install Docker
+
+ ```
+ $ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+ $ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+ $ sudo apt-get update
+ $ sudo apt-get install -y docker-ce
+ ```
+
+2. Build image
+
+ ```
+ $ sudo docker build -t electrum-sdist-builder-img contrib/build-linux/sdist
+ ```
+
+3. Build source tarballs
+
+ It's recommended to build from a fresh clone
+ (but you can skip this if reproducibility is not necessary).
+
+ ```
+ $ FRESH_CLONE=contrib/build-linux/sdist/fresh_clone && \
+ sudo rm -rf $FRESH_CLONE && \
+ umask 0022 && \
+ mkdir -p $FRESH_CLONE && \
+ cd $FRESH_CLONE && \
+ git clone https://github.com/spesmilo/electrum.git && \
+ cd electrum
+ ```
+
+ And then build from this directory:
+ ```
+ $ git checkout $REV
+ $ sudo docker run -it \
+ --name electrum-sdist-builder-cont \
+ -v $PWD:/opt/electrum \
+ --rm \
+ --workdir /opt/electrum/contrib/build-linux/sdist \
+ electrum-sdist-builder-img \
+ ./build.sh
+ ```
+4. The generated distributables are in `./dist`.
diff --git a/contrib/build-linux/sdist/build.sh b/contrib/build-linux/sdist/build.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -e
+
+PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.."
+CONTRIB="$PROJECT_ROOT/contrib"
+CONTRIB_SDIST="$CONTRIB/build-linux/sdist"
+DISTDIR="$PROJECT_ROOT/dist"
+
+. "$CONTRIB"/build_tools_util.sh
+
+# note that at least py3.7 is needed, to have https://bugs.python.org/issue30693
+python3 --version || fail "python interpreter not found"
+
+# upgrade to modern pip so that it knows the flags we need.
+# we will then install a pinned version of pip as part of requirements-sdist-build
+python3 -m pip install --upgrade pip
+
+info "Installing pinned requirements."
+python3 -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-sdist-build.txt
+
+
+"$CONTRIB"/make_packages || fail "make_packages failed"
+
+"$CONTRIB_SDIST"/make_tgz || fail "make_tgz failed"
+
+
+info "done."
+ls -la "$DISTDIR"
+sha256sum "$DISTDIR"/*
diff --git a/contrib/build-linux/sdist/make_tgz b/contrib/build-linux/sdist/make_tgz
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -e
+
+CONTRIB_SDIST="$(dirname "$(readlink -e "$0")")"
+CONTRIB="$CONTRIB_SDIST"/../..
+ROOT_FOLDER="$CONTRIB"/..
+PACKAGES="$ROOT_FOLDER"/packages/
+LOCALE="$ROOT_FOLDER"/electrum/locale/
+
+if [ ! -d "$PACKAGES" ]; then
+ echo "Run make_packages first!"
+ exit 1
+fi
+
+git submodule update --init
+
+(
+ rm -rf "$LOCALE"
+ cd "$CONTRIB/deterministic-build/electrum-locale/"
+ if ! which msgfmt > /dev/null 2>&1; then
+ echo "Please install gettext"
+ exit 1
+ fi
+ for i in ./locale/*; do
+ dir="$ROOT_FOLDER"/electrum/$i/LC_MESSAGES
+ mkdir -p $dir
+ msgfmt --output-file=$dir/electrum.mo $i/electrum.po || true
+ cp $i/electrum.po "$ROOT_FOLDER"/electrum/$i/electrum.po
+ done
+)
+
+(
+ cd "$ROOT_FOLDER"
+
+ echo "'git clean -fd' would delete the following files: >>>"
+ git clean -fd --dry-run
+ echo "<<<"
+
+ # we could build the kivy atlas potentially?
+ #(cd electrum/gui/kivy/; make theming) || echo "building kivy atlas failed! skipping."
+
+ find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} +
+
+ # note: .zip sdists would not be reproducible due to https://bugs.python.org/issue40963
+ TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar
+)
diff --git a/contrib/deterministic-build/requirements-sdist-build.txt b/contrib/deterministic-build/requirements-sdist-build.txt
@@ -0,0 +1,9 @@
+pip==20.1.1 \
+ --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
+ --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
+setuptools==46.4.0 \
+ --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
+ --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
+wheel==0.34.2 \
+ --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
+ --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e+
\ No newline at end of file
diff --git a/contrib/freeze_packages.sh b/contrib/freeze_packages.sh
@@ -20,7 +20,7 @@ which virtualenv > /dev/null 2>&1 || { echo "Please install virtualenv" && exit
${SYSTEM_PYTHON} -m hashin -h > /dev/null 2>&1 || { ${SYSTEM_PYTHON} -m pip install hashin; }
-for i in '' '-hw' '-binaries' '-wine-build' '-mac-build'; do
+for i in '' '-hw' '-binaries' '-wine-build' '-mac-build' '-sdist-build'; do
rm -rf "$venv_dir"
virtualenv -p ${SYSTEM_PYTHON} $venv_dir
diff --git a/contrib/make_packages b/contrib/make_packages
@@ -6,5 +6,6 @@ test -n "$CONTRIB" -a -d "$CONTRIB" || exit
rm "$CONTRIB"/../packages/ -r
#Install pure python modules in electrum directory
-python3 -m pip install -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages
+python3 -m pip install --no-dependencies --no-binary :all: \
+ -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages
diff --git a/contrib/make_tgz b/contrib/make_tgz
@@ -1,43 +0,0 @@
-#!/bin/bash
-
-set -e
-
-CONTRIB="$(dirname "$(readlink -e "$0")")"
-ROOT_FOLDER="$CONTRIB"/..
-PACKAGES="$ROOT_FOLDER"/packages/
-LOCALE="$ROOT_FOLDER"/electrum/locale/
-
-if [ ! -d "$PACKAGES" ]; then
- echo "Run make_packages first!"
- exit 1
-fi
-
-git submodule update --init
-
-(
- rm -rf "$LOCALE"
- cd "$CONTRIB/deterministic-build/electrum-locale/"
- if ! which msgfmt > /dev/null 2>&1; then
- echo "Please install gettext"
- exit 1
- fi
- for i in ./locale/*; do
- dir="$ROOT_FOLDER"/electrum/$i/LC_MESSAGES
- mkdir -p $dir
- msgfmt --output-file=$dir/electrum.mo $i/electrum.po || true
- cp $i/electrum.po "$ROOT_FOLDER"/electrum/$i/electrum.po
- done
-)
-
-(
- cd "$ROOT_FOLDER"
-
- echo "'git clean -fd' would delete the following files: >>>"
- git clean -fd --dry-run
- echo "<<<"
-
- # we could build the kivy atlas potentially?
- #(cd electrum/gui/kivy/; make theming) || echo "building kivy atlas failed! skipping."
-
- python3 setup.py --quiet sdist --format=zip,gztar
-)
diff --git a/contrib/requirements/requirements-sdist-build.txt b/contrib/requirements/requirements-sdist-build.txt
@@ -0,0 +1,3 @@
+# need modern versions of pip (and maybe other build tools), the one in apt had issues
+pip
+setuptools
