From 6d0ca4e25ef2a0962583697d6afe1fbf22aa44c8 Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Mon, 8 Oct 2018 15:53:22 +0200 Subject: [PATCH 1/7] Prepare Depends Packages for Gitian Scripts The gitian environment does not treat whitespaces in configure lines, like most other systems. The solution is to just remove them. --- contrib/depends/packages/libiconv.mk | 15 ++++++++++++++- contrib/depends/packages/sodium.mk | 2 ++ .../depends/patches/libiconv/fix-whitespace.patch | 13 +++++++++++++ .../depends/patches/sodium/fix-whitespace.patch | 13 +++++++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 contrib/depends/patches/libiconv/fix-whitespace.patch create mode 100644 contrib/depends/patches/sodium/fix-whitespace.patch diff --git a/contrib/depends/packages/libiconv.mk b/contrib/depends/packages/libiconv.mk index 87e30b208..dbcb28141 100644 --- a/contrib/depends/packages/libiconv.mk +++ b/contrib/depends/packages/libiconv.mk @@ -3,9 +3,22 @@ $(package)_version=1.15 $(package)_download_path=https://ftp.gnu.org/gnu/libiconv $(package)_file_name=libiconv-$($(package)_version).tar.gz $(package)_sha256_hash=ccf536620a45458d26ba83887a983b96827001e92a13847b45e4925cc8913178 +$(package)_patches=fix-whitespace.patch + +define $(package)_set_vars + $(package)_config_opts=--disable-nls + $(package)_config_opts=--enable-static + $(package)_config_opts=--disable-shared + $(package)_config_opts_linux=--with-pic +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub build-aux/ &&\ + patch -p1 < $($(package)_patch_dir)/fix-whitespace.patch +endef define $(package)_config_cmds - $($(package)_autoconf) --disable-nls --enable-static --disable-shared + $($(package)_autoconf) endef define $(package)_build_cmds diff --git a/contrib/depends/packages/sodium.mk b/contrib/depends/packages/sodium.mk index c38121bf7..35f444fd5 100644 --- a/contrib/depends/packages/sodium.mk +++ b/contrib/depends/packages/sodium.mk @@ -3,6 +3,7 @@ $(package)_version=1.0.15 $(package)_download_path=https://download.libsodium.org/libsodium/releases/ $(package)_file_name=libsodium-$($(package)_version).tar.gz $(package)_sha256_hash=fb6a9e879a2f674592e4328c5d9f79f082405ee4bb05cb6e679b90afe9e178f4 +$(package)_patches=fix-whitespace.patch define $(package)_set_vars $(package)_config_opts=--enable-static --disable-shared @@ -11,6 +12,7 @@ endef define $(package)_config_cmds ./autogen.sh &&\ + patch -p1 < $($(package)_patch_dir)/fix-whitespace.patch &&\ $($(package)_autoconf) $($(package)_config_opts) endef diff --git a/contrib/depends/patches/libiconv/fix-whitespace.patch b/contrib/depends/patches/libiconv/fix-whitespace.patch new file mode 100644 index 000000000..531364b45 --- /dev/null +++ b/contrib/depends/patches/libiconv/fix-whitespace.patch @@ -0,0 +1,13 @@ +diff --git a/preload/configure b/preload/configure +index aab5c77..e20b8f0 100755 +--- a/preload/configure ++++ b/preload/configure +@@ -588,7 +588,7 @@ MAKEFLAGS= + PACKAGE_NAME='libiconv' + PACKAGE_TARNAME='libiconv' + PACKAGE_VERSION='0' +-PACKAGE_STRING='libiconv 0' ++PACKAGE_STRING='libiconv0' + PACKAGE_BUGREPORT='' + PACKAGE_URL='' + diff --git a/contrib/depends/patches/sodium/fix-whitespace.patch b/contrib/depends/patches/sodium/fix-whitespace.patch new file mode 100644 index 000000000..c11838611 --- /dev/null +++ b/contrib/depends/patches/sodium/fix-whitespace.patch @@ -0,0 +1,13 @@ +diff --git a/configure b/configure +index b29f769..ca008ae 100755 +--- a/configure ++++ b/configure +@@ -591,7 +591,7 @@ MAKEFLAGS= + PACKAGE_NAME='libsodium' + PACKAGE_TARNAME='libsodium' + PACKAGE_VERSION='1.0.15' +-PACKAGE_STRING='libsodium 1.0.15' ++PACKAGE_STRING='libsodium' + PACKAGE_BUGREPORT='https://github.com/jedisct1/libsodium/issues' + PACKAGE_URL='https://github.com/jedisct1/libsodium' + From c2f178902aedeb3e862dc1029c5e2531a18e42bd Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Mon, 8 Oct 2018 15:55:38 +0200 Subject: [PATCH 2/7] Add gitian build script This adds a build script to run gitian builds for linux. The build script was copied from bitcoin and then adapted for monero. Build step documentation is outlined in the README in the contrib/gitian directory. --- contrib/gitian/README.md | 158 ++++++++++++++++++++++ contrib/gitian/gitian-build.py | 229 ++++++++++++++++++++++++++++++++ contrib/gitian/gitian-linux.yml | 162 ++++++++++++++++++++++ contrib/gitian/symbol-check.py | 163 +++++++++++++++++++++++ 4 files changed, 712 insertions(+) create mode 100644 contrib/gitian/README.md create mode 100755 contrib/gitian/gitian-build.py create mode 100644 contrib/gitian/gitian-linux.yml create mode 100755 contrib/gitian/symbol-check.py diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md new file mode 100644 index 000000000..1f7d158eb --- /dev/null +++ b/contrib/gitian/README.md @@ -0,0 +1,158 @@ +Gitian building +================ + +*Setup instructions for a Gitian build of Monero using a VM or physical system.* + +Gitian is the deterministic build process that is used to build the Bitcoin +Core executables. It provides a way to be reasonably sure that the +executables are really built from the git source. It also makes sure that +the same, tested dependencies are used and statically built into the executable. + +Multiple developers build the source code by following a specific descriptor +("recipe"), cryptographically sign the result, and upload the resulting signature. +These results are compared and only if they match, the build is accepted and provided +for download. + +More independent Gitian builders are needed, which is why this guide exists. +It is preferred you follow these steps yourself instead of using someone else's +VM image to avoid 'contaminating' the build. + +Table of Contents +------------------ + +Please note that these instructions have been forked from bitcoin's gitian build +instructions. Please also consult their documentation, when running into problems. +The signing is left as inherited from bitcoin at the moment, since building currently +still fails with libiconv. + +- [Preparing the Gitian builder host](#preparing-the-gitian-builder-host) +- [Getting and building the inputs](#getting-and-building-the-inputs) +- [Building Binaries](#building-bitcoin-core) +- [Signing externally](#signing-externally) +- [Uploading signatures](#uploading-signatures) + +Preparing the Gitian builder host +--------------------------------- + +The first step is to prepare the host environment that will be used to perform the Gitian builds. +This guide explains how to set up the environment, and how to start the builds. + +Gitian builds are for now executed on Ubuntu 18.04 "Bionic Beaver". Please run Ubuntu in either a VM, or on your physical machine. +You need to be logged in as the `gitianuser` in order to build gitian builds. If this user does not exist yet on your system, +create him. + +Note that a version of `lxc-execute` higher or equal to 2.1.1 is required. +You can check the version with `lxc-execute --version`. + +First we need to set up dependencies. Type/paste the following in the terminal: + +```bash +sudo apt-get install git ruby apt-cacher-ng qemu-utils debootstrap lxc python-cheetah parted kpartx bridge-utils make ubuntu-archive-keyring curl firewalld +``` + +Then set up LXC and the rest with the following, which is a complex jumble of settings and workarounds: + +```bash +sudo -s +# the version of lxc-start in Debian needs to run as root, so make sure +# that the build script can execute it without providing a password +echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-start" > /etc/sudoers.d/gitian-lxc +echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-execute" >> /etc/sudoers.d/gitian-lxc +# make /etc/rc.local script that sets up bridge between guest and host +echo '#!/bin/sh -e' > /etc/rc.local +echo 'brctl addbr br0' >> /etc/rc.local +echo 'ip addr add 10.0.3.1/24 broadcast 10.0.3.255 dev br0' >> /etc/rc.local +echo 'ip link set br0 up' >> /etc/rc.local +echo 'firewall-cmd --zone=trusted --add-interface=br0' >> /etc/rc.local +echo 'exit 0' >> /etc/rc.local +chmod +x /etc/rc.local +# make sure that USE_LXC is always set when logging in as gitianuser, +# and configure LXC IP addresses +echo 'export USE_LXC=1' >> /home/gitianuser/.profile +echo 'export GITIAN_HOST_IP=10.0.3.1' >> /home/gitianuser/.profile +echo 'export LXC_GUEST_IP=10.0.3.5' >> /home/gitianuser/.profile +reboot +``` + +This setup is required to enable networking in the container. + + +Manual and Building +------------------- +The instructions below use the automated script [gitian-build.py](https://github.com/betcoin/bitcoin/blob/master/contrib/gitian-build.py) which only works in Ubuntu. For manual steps and instructions for fully offline signing, see [this guide](./gitian-building/gitian-building-manual.md). + +MacOS code signing +------------------ +In order to sign builds for MacOS, you need to download the free SDK and extract a file. The steps are described [here](./gitian-building/gitian-building-mac-os-sdk.md). Alternatively, you can skip the OSX build by adding `--os=lw` below. + +Initial Gitian Setup +-------------------- +The `gitian-build.py` script will checkout different release tags, so it's best to copy it: + +```bash +cp monero/contrib/gitian/gitian-build.py . +``` + +You only need to do this once: + +``` +./gitian-build.py --setup fluffypony 0.0.20 +``` + +Where `fluffypony` is your Github name and `0.0.20` is the most recent tag (without `v`). + +In order to sign gitian builds on your host machine, which has your PGP key, fork the gitian.sigs repository and clone it on your host machine: + +``` +git clone git@github.com:bitcoin-core/gitian.sigs.git +git remote add satoshi git@github.com:satoshi/gitian.sigs.git +``` + +Build Binaries +----------------------------- +Windows and OSX have code signed binaries, but those won't be available until a few developers have gitian signed the non-codesigned binaries. + +To build the most recent tag: + + `./gitian-build.py --detach-sign --no-commit -b fluffypony 0.0.20` + +To speed up the build, use `-j 5 -m 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. + +If all went well, this produces a number of (uncommited) `.assert` files in the gitian.sigs repository. + +You need to copy these uncommited changes to your host machine, where you can sign them: + +``` +export NAME=satoshi +gpg --output $VERSION-linux/$NAME/bitcoin-linux-0.16-build.assert.sig --detach-sign 0.16.0rc1-linux/$NAME/bitcoin-linux-0.16-build.assert +gpg --output $VERSION-osx-unsigned/$NAME/bitcoin-osx-0.16-build.assert.sig --detach-sign 0.16.0rc1-osx-unsigned/$NAME/bitcoin-osx-0.16-build.assert +gpg --output $VERSION-win-unsigned/$NAME/bitcoin-win-0.16-build.assert.sig --detach-sign 0.16.0rc1-win-unsigned/$NAME/bitcoin-win-0.16-build.assert +``` + +Make a PR (both the `.assert` and `.assert.sig` files) to the +[bitcoin-core/gitian.sigs](https://github.com/bitcoin-core/gitian.sigs/) repository: + +``` +git checkout -b 0.0.20-not-codesigned +git commit -S -a -m "Add $NAME 0.0.20 non-code signed signatures" +git push --set-upstream $NAME 0.0.20 +``` + +You can also mail the files to Wladimir (laanwj@gmail.com) and he will commit them. + +```bash + gpg --detach-sign ${VERSION}-linux/${SIGNER}/bitcoin-linux-*-build.assert + gpg --detach-sign ${VERSION}-win-unsigned/${SIGNER}/bitcoin-win-*-build.assert + gpg --detach-sign ${VERSION}-osx-unsigned/${SIGNER}/bitcoin-osx-*-build.assert +``` + +You may have other .assert files as well (e.g. `signed` ones), in which case you should sign them too. You can see all of them by doing `ls ${VERSION}-*/${SIGNER}`. + +This will create the `.sig` files that can be committed together with the `.assert` files to assert your +Gitian build. + + + `./gitian-build.py --detach-sign -s satoshi 0.16.0rc1 --nocommit` + +Make another pull request for these. + diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py new file mode 100755 index 000000000..bd7e70a71 --- /dev/null +++ b/contrib/gitian/gitian-build.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 + +import argparse +import os +import subprocess +import sys + +def setup(): + global args, workdir + programs = ['ruby', 'git', 'apt-cacher-ng', 'make', 'wget'] + if args.kvm: + programs += ['python-vm-builder', 'qemu-kvm', 'qemu-utils'] + elif args.docker: + dockers = ['docker.io', 'docker-ce'] + for i in dockers: + return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) + if return_code == 0: + break + if return_code != 0: + print('Cannot find any way to install docker', file=sys.stderr) + exit(1) + else: + programs += ['lxc', 'debootstrap'] + subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) + if not os.path.isdir('gitian.sigs'): + subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/gitian.sigs.git']) + if not os.path.isdir('monero-detached-sigs'): + subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/bitcoin-detached-sigs.git']) + if not os.path.isdir('gitian-builder'): + subprocess.check_call(['git', 'clone', 'https://github.com/devrandom/gitian-builder.git']) + if not os.path.isdir('monero'): + subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/monero.git']) + os.chdir('gitian-builder') + make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64'] + if args.docker: + make_image_prog += ['--docker'] + elif not args.kvm: + make_image_prog += ['--lxc'] + subprocess.check_call(make_image_prog) + os.chdir(workdir) + if args.is_bionic and not args.kvm and not args.docker: + subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) + print('Reboot is required') + exit(0) + +def build(): + global args, workdir + + os.makedirs('monero-binaries/' + args.version, exist_ok=True) + print('\nBuilding Dependencies\n') + os.chdir('gitian-builder') + os.makedirs('inputs', exist_ok=True) + + subprocess.check_call(['wget', '-N', '-P', 'inputs', 'http://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz']) + subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch']) + subprocess.check_call(['make', '-C', '../monero/contrib/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) + + if args.linux: + print('\nCompiling ' + args.version + ' Linux') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-linux.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-linux.yml']) + subprocess.check_call('mv build/out/monero-*.tar.gz build/out/monero-*.tar.gz ../monero-binaries/'+args.version, shell=True) + + if args.windows: + print('\nCompiling ' + args.version + ' Windows') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-win.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-unsigned', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-win.yml']) + subprocess.check_call('mv build/out/monero*-win-unsigned.tar.gz inputs/monerowin-unsigned.tar.gz', shell=True) + subprocess.check_call('mv build/out/monero*.zip build/out/monero*.exe ../monerobinaries/'+args.version, shell=True) + + if args.macos: + print('\nCompiling ' + args.version + ' MacOS') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero'+args.commit, '--url', 'monero'+args.url, '../monero/contrib/gitian/gitian-osx.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-osx.yml']) + subprocess.check_call('mv build/out/monero*-osx-unsigned.tar.gz inputs/moneroosx-unsigned.tar.gz', shell=True) + subprocess.check_call('mv build/out/monero*.tar.gz build/out/monero*.dmg ../monerobinaries/'+args.version, shell=True) + + os.chdir(workdir) + + if args.commit_files: + print('\nCommitting '+args.version+' Unsigned Sigs\n') + os.chdir('gitian.sigs') + subprocess.check_call(['git', 'add', args.version+'-linux/'+args.signer]) + subprocess.check_call(['git', 'add', args.version+'-win-unsigned/'+args.signer]) + subprocess.check_call(['git', 'add', args.version+'-osx-unsigned/'+args.signer]) + subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer]) + os.chdir(workdir) + +def sign(): + global args, workdir + os.chdir('gitian-builder') + + if args.windows: + print('\nSigning ' + args.version + ' Windows') + subprocess.check_call(['bin/gbuild', '-i', '--commit', 'signature='+args.commit, '../monero/contrib/gitian/gitian-win-signer.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-signed', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-win-signer.yml']) + subprocess.check_call('mv build/out/monero*win64-setup.exe ../monerobinaries/'+args.version, shell=True) + subprocess.check_call('mv build/out/monero*win32-setup.exe ../monerobinaries/'+args.version, shell=True) + + if args.macos: + print('\nSigning ' + args.version + ' MacOS') + subprocess.check_call(['bin/gbuild', '-i', '--commit', 'signature='+args.commit, '../monero/contrib/gitian/gitian-osx-signer.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-signed', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-osx-signer.yml']) + subprocess.check_call('mv build/out/moneroosx-signed.dmg ../monerobinaries/'+args.version+'/monero'+args.version+'-osx.dmg', shell=True) + + os.chdir(workdir) + + if args.commit_files: + print('\nCommitting '+args.version+' Signed Sigs\n') + os.chdir('gitian.sigs') + subprocess.check_call(['git', 'add', args.version+'-win-signed/'+args.signer]) + subprocess.check_call(['git', 'add', args.version+'-osx-signed/'+args.signer]) + subprocess.check_call(['git', 'commit', '-a', '-m', 'Add '+args.version+' signed binary sigs for '+args.signer]) + os.chdir(workdir) + +def verify(): + global args, workdir + os.chdir('gitian-builder') + + print('\nVerifying v'+args.version+' Linux\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../monero/contrib/gitian/gitian-linux.yml']) + print('\nVerifying v'+args.version+' Windows\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../monero/contrib/gitian/gitian-win.yml']) + print('\nVerifying v'+args.version+' MacOS\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-unsigned', '../monero/contrib/gitian/gitian-osx.yml']) + print('\nVerifying v'+args.version+' Signed Windows\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-signed', '../monero/contrib/gitian/gitian-win-signer.yml']) + print('\nVerifying v'+args.version+' Signed MacOS\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-signed', '../monero/contrib/gitian/gitian-osx-signer.yml']) + + os.chdir(workdir) + +def main(): + global args, workdir + + parser = argparse.ArgumentParser(usage='%(prog)s [options] signer version') + parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') + parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request') + parser.add_argument('-u', '--url', dest='url', default='https://github.com/TheCharlatan/monero', help='Specify the URL of the repository. Default is %(default)s') + parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build') + parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build') + parser.add_argument('-s', '--sign', action='store_true', dest='sign', help='Make signed binaries for Windows and MacOS') + parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries') + parser.add_argument('-o', '--os', dest='os', default='lwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS') + parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s') + parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s') + parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC') + parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC') + parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Uses LXC. If you want to use KVM, use the --kvm option. Only works on Debian-based systems (Ubuntu, Debian)') + parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.') + parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git') + parser.add_argument('signer', help='GPG signer to sign each build assert file') + parser.add_argument('version', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified') + + args = parser.parse_args() + workdir = os.getcwd() + + args.linux = 'l' in args.os + args.windows = 'w' in args.os + args.macos = 'm' in args.os + + args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs']) + + if args.buildsign: + args.build=True + args.sign=True + + if args.kvm and args.docker: + raise Exception('Error: cannot have both kvm and docker') + + args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' + + # Set enviroment variable USE_LXC or USE_DOCKER, let gitian-builder know that we use lxc or docker + if args.docker: + os.environ['USE_DOCKER'] = '1' + elif not args.kvm: + os.environ['USE_LXC'] = '1' + if not 'GITIAN_HOST_IP' in os.environ.keys(): + os.environ['GITIAN_HOST_IP'] = '10.0.3.1' + if not 'LXC_GUEST_IP' in os.environ.keys(): + os.environ['LXC_GUEST_IP'] = '10.0.3.5' + + # Disable for MacOS if no SDK found + if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'): + print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') + args.macos = False + + script_name = os.path.basename(sys.argv[0]) + # Signer and version shouldn't be empty + if args.signer == '': + print(script_name+': Missing signer.') + print('Try '+script_name+' --help for more information') + exit(1) + if args.version == '': + print(script_name+': Missing version.') + print('Try '+script_name+' --help for more information') + exit(1) + + # Add leading 'v' for tags + if args.commit and args.pull: + raise Exception('Cannot have both commit and pull') + args.commit = ('' if args.commit else 'v') + args.version + + if args.setup: + setup() + + os.chdir('monero') + if args.pull: + subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) + os.chdir('../gitian-builder/inputs/monero') + subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) + args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True).strip() + args.version = 'pull-' + args.version + print(args.commit) + subprocess.check_call(['git', 'fetch']) + subprocess.check_call(['git', 'checkout', args.commit]) + os.chdir(workdir) + + if args.build: + build() + + if args.sign: + sign() + + if args.verify: + verify() + +if __name__ == '__main__': + main() diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml new file mode 100644 index 000000000..751fbadd3 --- /dev/null +++ b/contrib/gitian/gitian-linux.yml @@ -0,0 +1,162 @@ +--- +name: "monero-linux-0.18" +enable_cache: true +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "curl" +- "gperf" +- "gcc-7" +- "g++-7" +- "gcc" +- "g++" +- "gcc-7-aarch64-linux-gnu" +- "g++-7-aarch64-linux-gnu" +- "gcc-aarch64-linux-gnu" +- "g++-aarch64-linux-gnu" +- "binutils-aarch64-linux-gnu" +- "gcc-7-arm-linux-gnueabihf" +- "g++-7-arm-linux-gnueabihf" +- "gcc-arm-linux-gnueabihf" +- "g++-arm-linux-gnueabihf" +- "g++-7-multilib" +- "gcc-7-multilib" +- "binutils-arm-linux-gnueabihf" +- "binutils-gold" +- "git" +- "pkg-config" +- "build-essential" +- "autoconf" +- "libtool" +- "automake" +- "faketime" +- "bsdmainutils" +- "ca-certificates" +- "python" +- "cmake" +- "ccache" +- "protobuf-compiler" +- "libdbus-1-dev" +- "libharfbuzz-dev" +- "libprotobuf-dev" +- "python3-zmq" +remotes: +- "url": "https://github.com/TheCharlatan/monero.git" + "dir": "monero" +files: [] +script: | + + WRAP_DIR=$HOME/wrapped + HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu" + FAKETIME_HOST_PROGS="gcc g++" + FAKETIME_PROGS="date ar ranlib nm" + HOST_CFLAGS="-O2 -g" + HOST_CXXFLAGS="-O2 -g" + HOST_LDFLAGS=-static-libstdc++ + + export GZIP="-9n" + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export TZ="UTC" + export BUILD_DIR=`pwd` + mkdir -p ${WRAP_DIR} + if test -n "$GBUILD_CACHE_ENABLED"; then + export SOURCES_PATH=${GBUILD_COMMON_CACHE} + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + mkdir -p ${BASE_CACHE} ${SOURCES_PATH} + fi + + function create_global_faketime_wrappers { + for prog in ${FAKETIME_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} + echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${prog} + chmod +x ${WRAP_DIR}/${prog} + done + } + + function create_per-host_faketime_wrappers { + for i in $HOSTS; do + for prog in ${FAKETIME_HOST_PROGS}; do + if which ${i}-${prog}-7 + then + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog}-7 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + fi + done + done + } + + # Faketime for depends so intermediate results are comparable + export PATH_orig=${PATH} + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + export PATH=${WRAP_DIR}:${PATH} + + EXTRA_INCLUDES_BASE=$WRAP_DIR/extra_includes + mkdir -p $EXTRA_INCLUDES_BASE + + # x86 needs /usr/include/i386-linux-gnu/asm pointed to /usr/include/x86_64-linux-gnu/asm, + # but we can't write there. Instead, create a link here and force it to be included in the + # search paths by wrapping gcc/g++. + + mkdir -p $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu + rm -f $WRAP_DIR/extra_includes/i686-pc-linux-gnu/asm + ln -s /usr/include/x86_64-linux-gnu/asm $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu/asm + + for prog in gcc g++; do + rm -f ${WRAP_DIR}/${prog} + cat << EOF > ${WRAP_DIR}/${prog} + #!/usr/bin/env bash + REAL="`which -a ${prog}-7 | grep -v ${WRAP_DIR}/${prog} | head -1`" + for var in "\$@" + do + if [ "\$var" = "-m32" ]; then + export C_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" + export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" + break + fi + done + \$REAL \$@ + EOF + chmod +x ${WRAP_DIR}/${prog} + done + + cd monero + BASEPREFIX=`pwd`/contrib/depends + # Build dependencies for each host + for i in $HOSTS; do + EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" + if [ -d "$EXTRA_INCLUDES" ]; then + export HOST_ID_SALT="$EXTRA_INCLUDES" + fi + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" -j 4 V=1 + unset HOST_ID_SALT + done + + # Faketime for binaries + export PATH=${PATH_orig} + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + export PATH=${WRAP_DIR}:${PATH} + + ORIGPATH="$PATH" + # Extract the release tarball into a dir for each host and build + for i in ${HOSTS}; do + export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + mkdir build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake + make + DISTNAME=monero-${i}.tar.gz + tar -cvzf $DISTNAME bin/* + cp $DISTNAME $OUTDIR/ + cd .. + rm -rf build + done diff --git a/contrib/gitian/symbol-check.py b/contrib/gitian/symbol-check.py new file mode 100755 index 000000000..6808e77da --- /dev/null +++ b/contrib/gitian/symbol-check.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014 Wladimir J. van der Laan +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +A script to check that the (Linux) executables produced by gitian only contain +allowed gcc, glibc and libstdc++ version symbols. This makes sure they are +still compatible with the minimum supported Linux distribution versions. + +Example usage: + + find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py +''' +import subprocess +import re +import sys +import os + +# Debian 6.0.9 (Squeeze) has: +# +# - g++ version 4.4.5 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=g%2B%2B) +# - libc version 2.11.3 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libc6) +# - libstdc++ version 4.4.5 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libstdc%2B%2B6) +# +# Ubuntu 10.04.4 (Lucid Lynx) has: +# +# - g++ version 4.4.3 (http://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=lucid§ion=all) +# - libc version 2.11.1 (http://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=lucid§ion=all) +# - libstdc++ version 4.4.3 (http://packages.ubuntu.com/search?suite=lucid§ion=all&arch=any&keywords=libstdc%2B%2B&searchon=names) +# +# Taking the minimum of these as our target. +# +# According to GNU ABI document (http://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to: +# GCC 4.4.0: GCC_4.4.0 +# GCC 4.4.2: GLIBCXX_3.4.13, CXXABI_1.3.3 +# (glibc) GLIBC_2_11 +# +MAX_VERSIONS = { +'GCC': (4,4,0), +'CXXABI': (1,3,3), +'GLIBCXX': (3,4,13), +'GLIBC': (2,11) +} +# See here for a description of _IO_stdin_used: +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 + +# Ignore symbols that are exported as part of every executable +IGNORE_EXPORTS = { +'_edata', '_end', '_init', '__bss_start', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr' +} +READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') +CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') +# Allowed NEEDED libraries +ALLOWED_LIBRARIES = { +# bitcoind and bitcoin-qt +'libgcc_s.so.1', # GCC base support +'libc.so.6', # C library +'libpthread.so.0', # threading +'libanl.so.1', # DNS resolve +'libm.so.6', # math library +'librt.so.1', # real-time (clock) +'ld-linux-x86-64.so.2', # 64-bit dynamic linker +'ld-linux.so.2', # 32-bit dynamic linker +# bitcoin-qt only +'libX11-xcb.so.1', # part of X11 +'libX11.so.6', # part of X11 +'libxcb.so.1', # part of X11 +'libfontconfig.so.1', # font support +'libfreetype.so.6', # font parsing +'libdl.so.2' # programming interface to dynamic linker +} + +class CPPFilt(object): + ''' + Demangle C++ symbol names. + + Use a pipe to the 'c++filt' command. + ''' + def __init__(self): + self.proc = subprocess.Popen(CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) + + def __call__(self, mangled): + self.proc.stdin.write(mangled + '\n') + self.proc.stdin.flush() + return self.proc.stdout.readline().rstrip() + + def close(self): + self.proc.stdin.close() + self.proc.stdout.close() + self.proc.wait() + +def read_symbols(executable, imports=True): + ''' + Parse an ELF executable and return a list of (symbol,version) tuples + for dynamic, imported symbols. + ''' + p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Could not read symbols for %s: %s' % (executable, stderr.strip())) + syms = [] + for line in stdout.splitlines(): + line = line.split() + if len(line)>7 and re.match('[0-9]+:$', line[0]): + (sym, _, version) = line[7].partition('@') + is_import = line[6] == 'UND' + if version.startswith('@'): + version = version[1:] + if is_import == imports: + syms.append((sym, version)) + return syms + +def check_version(max_versions, version): + if '_' in version: + (lib, _, ver) = version.rpartition('_') + else: + lib = version + ver = '0' + ver = tuple([int(x) for x in ver.split('.')]) + if not lib in max_versions: + return False + return ver <= max_versions[lib] + +def read_libraries(filename): + p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + libraries = [] + for line in stdout.splitlines(): + tokens = line.split() + if len(tokens)>2 and tokens[1] == '(NEEDED)': + match = re.match('^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) + if match: + libraries.append(match.group(1)) + else: + raise ValueError('Unparseable (NEEDED) specification') + return libraries + +if __name__ == '__main__': + cppfilt = CPPFilt() + retval = 0 + for filename in sys.argv[1:]: + # Check imported symbols + for sym,version in read_symbols(filename, True): + if version and not check_version(MAX_VERSIONS, version): + print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym), version)) + retval = 1 + # Check exported symbols + for sym,version in read_symbols(filename, False): + if sym in IGNORE_EXPORTS: + continue + print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym))) + retval = 1 + # Check dependency libraries + for library_name in read_libraries(filename): + if library_name not in ALLOWED_LIBRARIES: + print('%s: NEEDED library %s is not allowed' % (filename, library_name)) + retval = 1 + + sys.exit(retval) + + From f2127f9dca29c8b1e0a51bebeee5e8e49e6f4876 Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Mon, 8 Oct 2018 23:14:46 +0200 Subject: [PATCH 3/7] Add checksums for download tools The signature prepare tool and the gitian-builder git repo should be checked for their content. For this purpose, checkout the gitian-builder repo at a specific commit and take the sha256sum of the osslsigncode tool. --- contrib/gitian/gitian-build.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py index bd7e70a71..2f4c93625 100755 --- a/contrib/gitian/gitian-build.py +++ b/contrib/gitian/gitian-build.py @@ -24,13 +24,12 @@ def setup(): subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) if not os.path.isdir('gitian.sigs'): subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/gitian.sigs.git']) - if not os.path.isdir('monero-detached-sigs'): - subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/bitcoin-detached-sigs.git']) if not os.path.isdir('gitian-builder'): subprocess.check_call(['git', 'clone', 'https://github.com/devrandom/gitian-builder.git']) if not os.path.isdir('monero'): - subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/monero.git']) + subprocess.check_call(['git', 'clone', 'https://github.com/monero-project/monero.git']) os.chdir('gitian-builder') + subprocess.check_call(['git', 'checkout', '963322de8420c50502c4cc33d4d7c0d84437b576']) make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64'] if args.docker: make_image_prog += ['--docker'] @@ -51,8 +50,10 @@ def build(): os.chdir('gitian-builder') os.makedirs('inputs', exist_ok=True) - subprocess.check_call(['wget', '-N', '-P', 'inputs', 'http://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz']) + subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz']) subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch']) + subprocess.check_output(["echo 'a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 inputs/osslsigncode-Backports-to-1.7.1.patch' | sha256sum -c"], shell=True) + subprocess.check_output(["echo 'f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 inputs/osslsigncode-1.7.1.tar.gz' | sha256sum -c"], shell=True) subprocess.check_call(['make', '-C', '../monero/contrib/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) if args.linux: From fed4e59886c3410429f9625caff0298ac7523bcb Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Tue, 9 Oct 2018 11:12:32 +0200 Subject: [PATCH 4/7] Change gitian.sigs repo from bitcoin-core to monero-project remote host --- contrib/gitian/gitian-build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py index 2f4c93625..0a93dbf5b 100755 --- a/contrib/gitian/gitian-build.py +++ b/contrib/gitian/gitian-build.py @@ -23,7 +23,7 @@ def setup(): programs += ['lxc', 'debootstrap'] subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) if not os.path.isdir('gitian.sigs'): - subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/gitian.sigs.git']) + subprocess.check_call(['git', 'clone', 'https://github.com/monero-project/gitian.sigs.git']) if not os.path.isdir('gitian-builder'): subprocess.check_call(['git', 'clone', 'https://github.com/devrandom/gitian-builder.git']) if not os.path.isdir('monero'): From d147d24058ef4d4e25ff0603b6bc89f2d4095874 Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Tue, 9 Oct 2018 22:24:49 +0200 Subject: [PATCH 5/7] Add windows descriptor to gitian descriptors Windows is built with a seperate descriptor to handle additional changes that need to be done to the end binary. Consolidate the gitian-build script for this change. --- contrib/gitian/gitian-build.py | 8 +- contrib/gitian/gitian-linux.yml | 6 +- contrib/gitian/gitian-win.yml | 135 ++++++++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 contrib/gitian/gitian-win.yml diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py index 0a93dbf5b..feb193a32 100755 --- a/contrib/gitian/gitian-build.py +++ b/contrib/gitian/gitian-build.py @@ -60,20 +60,20 @@ def build(): print('\nCompiling ' + args.version + ' Linux') subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-linux.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-linux.yml']) - subprocess.check_call('mv build/out/monero-*.tar.gz build/out/monero-*.tar.gz ../monero-binaries/'+args.version, shell=True) + subprocess.check_call('mv build/out/monero-*.tar.gz ../monero-binaries/'+args.version, shell=True) if args.windows: print('\nCompiling ' + args.version + ' Windows') subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-win.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-unsigned', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-win.yml']) - subprocess.check_call('mv build/out/monero*-win-unsigned.tar.gz inputs/monerowin-unsigned.tar.gz', shell=True) - subprocess.check_call('mv build/out/monero*.zip build/out/monero*.exe ../monerobinaries/'+args.version, shell=True) + subprocess.check_call('mv build/out/monero*-unsigned.tar.gz inputs/', shell=True) + subprocess.check_call('mv build/out/monero*.zip ../monerobinaries/'+args.version, shell=True) if args.macos: print('\nCompiling ' + args.version + ' MacOS') subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero'+args.commit, '--url', 'monero'+args.url, '../monero/contrib/gitian/gitian-osx.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-osx.yml']) - subprocess.check_call('mv build/out/monero*-osx-unsigned.tar.gz inputs/moneroosx-unsigned.tar.gz', shell=True) + subprocess.check_call('mv build/out/monero*-osx-unsigned.tar.gz inputs/', shell=True) subprocess.check_call('mv build/out/monero*.tar.gz build/out/monero*.dmg ../monerobinaries/'+args.version, shell=True) os.chdir(workdir) diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml index 751fbadd3..e64e28bbc 100644 --- a/contrib/gitian/gitian-linux.yml +++ b/contrib/gitian/gitian-linux.yml @@ -154,9 +154,9 @@ script: | mkdir build && cd build cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake make - DISTNAME=monero-${i}.tar.gz - tar -cvzf $DISTNAME bin/* - cp $DISTNAME $OUTDIR/ + DISTNAME=monero-${i} + mv bin ${DISTNAME} + find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz cd .. rm -rf build done diff --git a/contrib/gitian/gitian-win.yml b/contrib/gitian/gitian-win.yml new file mode 100644 index 000000000..7ae8c58f4 --- /dev/null +++ b/contrib/gitian/gitian-win.yml @@ -0,0 +1,135 @@ +--- +name: "bitcoin-win-0.18" +enable_cache: true +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "curl" +- "g++" +- "git" +- "pkg-config" +- "autoconf" +- "libtool" +- "automake" +- "faketime" +- "bsdmainutils" +- "mingw-w64" +- "g++-mingw-w64" +- "zip" +- "ca-certificates" +- "python" +- "rename" +- "cmake" +remotes: +- "url": "https://github.com/monero-project/monero.git" + "dir": "monero" +files: [] +script: | + WRAP_DIR=$HOME/wrapped + HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" + FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" + FAKETIME_PROGS="date zip" + HOST_CFLAGS="-O2 -g" + HOST_CXXFLAGS="-O2 -g" + + export GZIP="-9n" + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export TZ="UTC" + export BUILD_DIR=`pwd` + mkdir -p ${WRAP_DIR} + if test -n "$GBUILD_CACHE_ENABLED"; then + export SOURCES_PATH=${GBUILD_COMMON_CACHE} + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + mkdir -p ${BASE_CACHE} ${SOURCES_PATH} + fi + + function create_global_faketime_wrappers { + for prog in ${FAKETIME_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} + echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${prog} + chmod +x ${WRAP_DIR}/${prog} + done + } + + function create_per-host_faketime_wrappers { + for i in $HOSTS; do + for prog in ${FAKETIME_HOST_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + done + done + } + + function create_per-host_linker_wrapper { + # This is only needed for trusty, as the mingw linker leaks a few bytes of + # heap, causing non-determinism. See discussion in https://github.com/bitcoin/bitcoin/pull/6900 + for i in $HOSTS; do + mkdir -p ${WRAP_DIR}/${i} + for prog in collect2; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}/${prog} + REAL=$(${i}-gcc -print-prog-name=${prog}) + echo "export MALLOC_PERTURB_=255" >> ${WRAP_DIR}/${i}/${prog} + echo "${REAL} \$@" >> $WRAP_DIR/${i}/${prog} + chmod +x ${WRAP_DIR}/${i}/${prog} + done + for prog in gcc g++; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + done + done + } + + # Faketime for depends so intermediate results are comparable + export PATH_orig=${PATH} + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_linker_wrapper "2000-01-01 12:00:00" + export PATH=${WRAP_DIR}:${PATH} + + cd monero + BASEPREFIX=`pwd`/contrib/depends + # Build dependencies for each host + for i in $HOSTS; do + EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" + if [ -d "$EXTRA_INCLUDES" ]; then + export HOST_ID_SALT="$EXTRA_INCLUDES" + fi + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" -j 4 V=1 + unset HOST_ID_SALT + done + + # Faketime for binaries + export PATH=${PATH_orig} + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + export PATH=${WRAP_DIR}:${PATH} + + ORIGPATH="$PATH" + # Run cmake and make, for each create a new build/ directory, + # compile from there, archive, export and delete the archive again + for i in ${HOSTS}; do + export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + mkdir build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake + make + DISTNAME=monero-${i} + mv bin ${DISTNAME} + find ${DISTNAME}/ | sort | zip -X@ ${OUTDIR}/${DISTNAME}.zip + find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-unsigned.tar.gz + cd .. && rm -rf build + done + From 9617fad0704dbca9b083409d5e7ef0319cf0806d Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Wed, 10 Oct 2018 00:16:55 +0200 Subject: [PATCH 6/7] Add OSX gitian descriptor --- contrib/gitian/gitian-build.py | 24 ++----- contrib/gitian/gitian-linux.yml | 4 +- contrib/gitian/gitian-osx.yml | 114 ++++++++++++++++++++++++++++++++ contrib/gitian/gitian-win.yml | 1 - 4 files changed, 120 insertions(+), 23 deletions(-) create mode 100644 contrib/gitian/gitian-osx.yml diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py index feb193a32..eb138e41a 100755 --- a/contrib/gitian/gitian-build.py +++ b/contrib/gitian/gitian-build.py @@ -66,15 +66,13 @@ def build(): print('\nCompiling ' + args.version + ' Windows') subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-win.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-unsigned', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-win.yml']) - subprocess.check_call('mv build/out/monero*-unsigned.tar.gz inputs/', shell=True) - subprocess.check_call('mv build/out/monero*.zip ../monerobinaries/'+args.version, shell=True) + subprocess.check_call('mv build/out/monero*.zip ../monero-binaries/'+args.version, shell=True) if args.macos: print('\nCompiling ' + args.version + ' MacOS') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero'+args.commit, '--url', 'monero'+args.url, '../monero/contrib/gitian/gitian-osx.yml']) + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero'+args.url, '../monero/contrib/gitian/gitian-osx.yml']) subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-osx.yml']) - subprocess.check_call('mv build/out/monero*-osx-unsigned.tar.gz inputs/', shell=True) - subprocess.check_call('mv build/out/monero*.tar.gz build/out/monero*.dmg ../monerobinaries/'+args.version, shell=True) + subprocess.check_call('mv build/out/monero*.tar.gz ../monero-binaries/'+args.version, shell=True) os.chdir(workdir) @@ -90,20 +88,6 @@ def build(): def sign(): global args, workdir os.chdir('gitian-builder') - - if args.windows: - print('\nSigning ' + args.version + ' Windows') - subprocess.check_call(['bin/gbuild', '-i', '--commit', 'signature='+args.commit, '../monero/contrib/gitian/gitian-win-signer.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-signed', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-win-signer.yml']) - subprocess.check_call('mv build/out/monero*win64-setup.exe ../monerobinaries/'+args.version, shell=True) - subprocess.check_call('mv build/out/monero*win32-setup.exe ../monerobinaries/'+args.version, shell=True) - - if args.macos: - print('\nSigning ' + args.version + ' MacOS') - subprocess.check_call(['bin/gbuild', '-i', '--commit', 'signature='+args.commit, '../monero/contrib/gitian/gitian-osx-signer.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-signed', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-osx-signer.yml']) - subprocess.check_call('mv build/out/moneroosx-signed.dmg ../monerobinaries/'+args.version+'/monero'+args.version+'-osx.dmg', shell=True) - os.chdir(workdir) if args.commit_files: @@ -137,7 +121,7 @@ def main(): parser = argparse.ArgumentParser(usage='%(prog)s [options] signer version') parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request') - parser.add_argument('-u', '--url', dest='url', default='https://github.com/TheCharlatan/monero', help='Specify the URL of the repository. Default is %(default)s') + parser.add_argument('-u', '--url', dest='url', default='https://github.com/monero-project/monero', help='Specify the URL of the repository. Default is %(default)s') parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build') parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build') parser.add_argument('-s', '--sign', action='store_true', dest='sign', help='Make signed binaries for Windows and MacOS') diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml index e64e28bbc..6c338f1da 100644 --- a/contrib/gitian/gitian-linux.yml +++ b/contrib/gitian/gitian-linux.yml @@ -43,7 +43,7 @@ packages: - "libprotobuf-dev" - "python3-zmq" remotes: -- "url": "https://github.com/TheCharlatan/monero.git" +- "url": "https://github.com/monero-project/monero.git" "dir": "monero" files: [] script: | @@ -148,7 +148,7 @@ script: | export PATH=${WRAP_DIR}:${PATH} ORIGPATH="$PATH" - # Extract the release tarball into a dir for each host and build + # Build in a new dir for each host for i in ${HOSTS}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir build && cd build diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml new file mode 100644 index 000000000..d9ebf95d3 --- /dev/null +++ b/contrib/gitian/gitian-osx.yml @@ -0,0 +1,114 @@ +--- +name: "bitcoin-osx-0.18" +enable_cache: true +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "ca-certificates" +- "curl" +- "g++" +- "git" +- "pkg-config" +- "autoconf" +- "librsvg2-bin" +- "libtiff-tools" +- "libtool" +- "automake" +- "faketime" +- "bsdmainutils" +- "cmake" +- "imagemagick" +- "libcap-dev" +- "libz-dev" +- "libbz2-dev" +- "python" +- "python-dev" +- "python-setuptools" +- "fonts-tuffy" +remotes: +- "url": "https://github.com/monero-project/monero.git" + "dir": "monero" +files: +- "MacOSX10.11.sdk.tar.gz" +script: | + WRAP_DIR=$HOME/wrapped + HOSTS="x86_64-apple-darwin11" + FAKETIME_HOST_PROGS="" + FAKETIME_PROGS="ar ranlib date dmg genisoimage" + + export GZIP="-9n" + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export TZ="UTC" + export BUILD_DIR=`pwd` + mkdir -p ${WRAP_DIR} + if test -n "$GBUILD_CACHE_ENABLED"; then + export SOURCES_PATH=${GBUILD_COMMON_CACHE} + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + mkdir -p ${BASE_CACHE} ${SOURCES_PATH} + fi + + export ZERO_AR_DATE=1 + + function create_global_faketime_wrappers { + for prog in ${FAKETIME_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} + echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${prog} + chmod +x ${WRAP_DIR}/${prog} + done + } + + function create_per-host_faketime_wrappers { + for i in $HOSTS; do + for prog in ${FAKETIME_HOST_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + done + done + } + + # Faketime for depends so intermediate results are comparable + export PATH_orig=${PATH} + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + export PATH=${WRAP_DIR}:${PATH} + + cd monero + BASEPREFIX=`pwd`/contrib/depends + + mkdir -p ${BASEPREFIX}/SDKs + tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz + + # Build dependencies for each host + for i in $HOSTS; do + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" + done + + # Faketime for binaries + export PATH=${PATH_orig} + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + export PATH=${WRAP_DIR}:${PATH} + + ORIGPATH="$PATH" + # Build in a new dir for each host + for i in ${HOSTS}; do + export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + mkdir build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake + make + DISTNAME=monero-${i} + mv bin ${DISTNAME} + find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz + cd .. + rm -rf build + done + diff --git a/contrib/gitian/gitian-win.yml b/contrib/gitian/gitian-win.yml index 7ae8c58f4..74c757c11 100644 --- a/contrib/gitian/gitian-win.yml +++ b/contrib/gitian/gitian-win.yml @@ -129,7 +129,6 @@ script: | DISTNAME=monero-${i} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | zip -X@ ${OUTDIR}/${DISTNAME}.zip - find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-unsigned.tar.gz cd .. && rm -rf build done From 8f96c718bc5b3844904b04868d5605778af41c39 Mon Sep 17 00:00:00 2001 From: TheCharlatan Date: Thu, 11 Oct 2018 02:05:00 +0200 Subject: [PATCH 7/7] Adapt Readme and script to monero gitian build signing The sigs should be produced in a seperate step by default. Remove windows and osx sig options that are not needed for monero. --- contrib/gitian/README.md | 78 ++++++++++++++------------------- contrib/gitian/gitian-build.py | 36 +++------------ contrib/gitian/gitian-linux.yml | 2 +- contrib/gitian/gitian-osx.yml | 2 +- contrib/gitian/gitian-win.yml | 2 +- 5 files changed, 43 insertions(+), 77 deletions(-) diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md index 1f7d158eb..4bd326f22 100644 --- a/contrib/gitian/README.md +++ b/contrib/gitian/README.md @@ -3,8 +3,8 @@ Gitian building *Setup instructions for a Gitian build of Monero using a VM or physical system.* -Gitian is the deterministic build process that is used to build the Bitcoin -Core executables. It provides a way to be reasonably sure that the +Gitian is the deterministic build process that is used to build the Monero CLI +executables. It provides a way to be reasonably sure that the executables are really built from the git source. It also makes sure that the same, tested dependencies are used and statically built into the executable. @@ -22,8 +22,7 @@ Table of Contents Please note that these instructions have been forked from bitcoin's gitian build instructions. Please also consult their documentation, when running into problems. -The signing is left as inherited from bitcoin at the moment, since building currently -still fails with libiconv. +The signing is left as inherited from bitcoin at the moment. - [Preparing the Gitian builder host](#preparing-the-gitian-builder-host) - [Getting and building the inputs](#getting-and-building-the-inputs) @@ -37,9 +36,10 @@ Preparing the Gitian builder host The first step is to prepare the host environment that will be used to perform the Gitian builds. This guide explains how to set up the environment, and how to start the builds. -Gitian builds are for now executed on Ubuntu 18.04 "Bionic Beaver". Please run Ubuntu in either a VM, or on your physical machine. +Gitian builds are for now executed on Ubuntu 18.04 "Bionic Beaver". A solution is being worked on to run +it in docker in the future. Please run Ubuntu in either a VM, or on your physical machine. You need to be logged in as the `gitianuser` in order to build gitian builds. If this user does not exist yet on your system, -create him. +create it. Note that a version of `lxc-execute` higher or equal to 2.1.1 is required. You can check the version with `lxc-execute --version`. @@ -79,11 +79,8 @@ This setup is required to enable networking in the container. Manual and Building ------------------- -The instructions below use the automated script [gitian-build.py](https://github.com/betcoin/bitcoin/blob/master/contrib/gitian-build.py) which only works in Ubuntu. For manual steps and instructions for fully offline signing, see [this guide](./gitian-building/gitian-building-manual.md). - -MacOS code signing ------------------- -In order to sign builds for MacOS, you need to download the free SDK and extract a file. The steps are described [here](./gitian-building/gitian-building-mac-os-sdk.md). Alternatively, you can skip the OSX build by adding `--os=lw` below. +The instructions below use the automated script [gitian-build.py](https://github.com/betcoin/bitcoin/blob/master/contrib/gitian-build.py) which only works in Ubuntu. +It calls all available descriptors. Help for the build steps taken can be accessed with `./gitian-build.py --help`. Initial Gitian Setup -------------------- @@ -93,66 +90,57 @@ The `gitian-build.py` script will checkout different release tags, so it's best cp monero/contrib/gitian/gitian-build.py . ``` -You only need to do this once: +Setup the required environment, you only need to do this once: ``` -./gitian-build.py --setup fluffypony 0.0.20 +./gitian-build.py --setup fluffypony 0.14.0 ``` -Where `fluffypony` is your Github name and `0.0.20` is the most recent tag (without `v`). +Where `fluffypony` is your Github name and `0.14.0` is the version tag you want to build (without `v`). -In order to sign gitian builds on your host machine, which has your PGP key, fork the gitian.sigs repository and clone it on your host machine: +While gitian and this build script does provide a way for you to sign the build directly, it is recommended to sign in a seperate step. +This script is only there for convenience. Seperate steps for building can still be taken. +In order to sign gitian builds on your host machine, which has your PGP key, +fork the gitian.sigs repository and clone it on your host machine, +or pass the signed assert file back to your build machine. ``` -git clone git@github.com:bitcoin-core/gitian.sigs.git -git remote add satoshi git@github.com:satoshi/gitian.sigs.git +git clone git@github.com:monero-project/gitian.sigs.git +git remote add fluffypony git@github.com:fluffypony/gitian.sigs.git ``` Build Binaries ----------------------------- -Windows and OSX have code signed binaries, but those won't be available until a few developers have gitian signed the non-codesigned binaries. - To build the most recent tag: - `./gitian-build.py --detach-sign --no-commit -b fluffypony 0.0.20` + `./gitian-build.py --detach-sign --no-commit -b fluffypony 0.14.0` To speed up the build, use `-j 5 -m 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. If all went well, this produces a number of (uncommited) `.assert` files in the gitian.sigs repository. -You need to copy these uncommited changes to your host machine, where you can sign them: +If you do detached, offline signing, you need to copy these uncommited changes to your host machine, where you can sign them. For example: ``` -export NAME=satoshi -gpg --output $VERSION-linux/$NAME/bitcoin-linux-0.16-build.assert.sig --detach-sign 0.16.0rc1-linux/$NAME/bitcoin-linux-0.16-build.assert -gpg --output $VERSION-osx-unsigned/$NAME/bitcoin-osx-0.16-build.assert.sig --detach-sign 0.16.0rc1-osx-unsigned/$NAME/bitcoin-osx-0.16-build.assert -gpg --output $VERSION-win-unsigned/$NAME/bitcoin-win-0.16-build.assert.sig --detach-sign 0.16.0rc1-win-unsigned/$NAME/bitcoin-win-0.16-build.assert +export NAME=fluffypony +export VERSION=0.14 +gpg --output $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert.sig --detach-sign $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert +gpg --output $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert.sig --detach-sign $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert +gpg --output $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert.sig --detach-sign $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert ``` -Make a PR (both the `.assert` and `.assert.sig` files) to the -[bitcoin-core/gitian.sigs](https://github.com/bitcoin-core/gitian.sigs/) repository: +Make a pull request (both the `.assert` and `.assert.sig` files) to the +[monero-project/gitian.sigs](https://github.com/monero-project/gitian.sigs/) repository: ``` -git checkout -b 0.0.20-not-codesigned -git commit -S -a -m "Add $NAME 0.0.20 non-code signed signatures" -git push --set-upstream $NAME 0.0.20 +git checkout -b 0.14.0 +git commit -S -a -m "Add $NAME 0.14.0" +git push --set-upstream $NAME 0.14.0 ``` -You can also mail the files to Wladimir (laanwj@gmail.com) and he will commit them. - ```bash - gpg --detach-sign ${VERSION}-linux/${SIGNER}/bitcoin-linux-*-build.assert - gpg --detach-sign ${VERSION}-win-unsigned/${SIGNER}/bitcoin-win-*-build.assert - gpg --detach-sign ${VERSION}-osx-unsigned/${SIGNER}/bitcoin-osx-*-build.assert + gpg --detach-sign ${VERSION}-linux/${SIGNER}/monero-linux-*-build.assert + gpg --detach-sign ${VERSION}-win-unsigned/${SIGNER}/monero-win-*-build.assert + gpg --detach-sign ${VERSION}-osx-unsigned/${SIGNER}/monero-osx-*-build.assert ``` -You may have other .assert files as well (e.g. `signed` ones), in which case you should sign them too. You can see all of them by doing `ls ${VERSION}-*/${SIGNER}`. - -This will create the `.sig` files that can be committed together with the `.assert` files to assert your -Gitian build. - - - `./gitian-build.py --detach-sign -s satoshi 0.16.0rc1 --nocommit` - -Make another pull request for these. - diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py index eb138e41a..99c64e9dd 100755 --- a/contrib/gitian/gitian-build.py +++ b/contrib/gitian/gitian-build.py @@ -65,13 +65,13 @@ def build(): if args.windows: print('\nCompiling ' + args.version + ' Windows') subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-win.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-unsigned', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-win.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-win.yml']) subprocess.check_call('mv build/out/monero*.zip ../monero-binaries/'+args.version, shell=True) if args.macos: print('\nCompiling ' + args.version + ' MacOS') subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero'+args.url, '../monero/contrib/gitian/gitian-osx.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-osx.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-osx.yml']) subprocess.check_call('mv build/out/monero*.tar.gz ../monero-binaries/'+args.version, shell=True) os.chdir(workdir) @@ -80,24 +80,11 @@ def build(): print('\nCommitting '+args.version+' Unsigned Sigs\n') os.chdir('gitian.sigs') subprocess.check_call(['git', 'add', args.version+'-linux/'+args.signer]) - subprocess.check_call(['git', 'add', args.version+'-win-unsigned/'+args.signer]) - subprocess.check_call(['git', 'add', args.version+'-osx-unsigned/'+args.signer]) + subprocess.check_call(['git', 'add', args.version+'-win/'+args.signer]) + subprocess.check_call(['git', 'add', args.version+'-osx/'+args.signer]) subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer]) os.chdir(workdir) -def sign(): - global args, workdir - os.chdir('gitian-builder') - os.chdir(workdir) - - if args.commit_files: - print('\nCommitting '+args.version+' Signed Sigs\n') - os.chdir('gitian.sigs') - subprocess.check_call(['git', 'add', args.version+'-win-signed/'+args.signer]) - subprocess.check_call(['git', 'add', args.version+'-osx-signed/'+args.signer]) - subprocess.check_call(['git', 'commit', '-a', '-m', 'Add '+args.version+' signed binary sigs for '+args.signer]) - os.chdir(workdir) - def verify(): global args, workdir os.chdir('gitian-builder') @@ -105,14 +92,9 @@ def verify(): print('\nVerifying v'+args.version+' Linux\n') subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../monero/contrib/gitian/gitian-linux.yml']) print('\nVerifying v'+args.version+' Windows\n') - subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../monero/contrib/gitian/gitian-win.yml']) + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win', '../monero/contrib/gitian/gitian-win.yml']) print('\nVerifying v'+args.version+' MacOS\n') - subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-unsigned', '../monero/contrib/gitian/gitian-osx.yml']) - print('\nVerifying v'+args.version+' Signed Windows\n') - subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-signed', '../monero/contrib/gitian/gitian-win-signer.yml']) - print('\nVerifying v'+args.version+' Signed MacOS\n') - subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-signed', '../monero/contrib/gitian/gitian-osx-signer.yml']) - + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx', '../monero/contrib/gitian/gitian-osx.yml']) os.chdir(workdir) def main(): @@ -124,7 +106,6 @@ def main(): parser.add_argument('-u', '--url', dest='url', default='https://github.com/monero-project/monero', help='Specify the URL of the repository. Default is %(default)s') parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build') parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build') - parser.add_argument('-s', '--sign', action='store_true', dest='sign', help='Make signed binaries for Windows and MacOS') parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries') parser.add_argument('-o', '--os', dest='os', default='lwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS') parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s') @@ -184,7 +165,7 @@ def main(): # Add leading 'v' for tags if args.commit and args.pull: raise Exception('Cannot have both commit and pull') - args.commit = ('' if args.commit else 'v') + args.version + args.commit = ('' if args.commit else) + args.version if args.setup: setup() @@ -204,9 +185,6 @@ def main(): if args.build: build() - if args.sign: - sign() - if args.verify: verify() diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml index 6c338f1da..473a7720d 100644 --- a/contrib/gitian/gitian-linux.yml +++ b/contrib/gitian/gitian-linux.yml @@ -1,5 +1,5 @@ --- -name: "monero-linux-0.18" +name: "monero-linux-0.14" enable_cache: true suites: - "bionic" diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml index d9ebf95d3..a6fcff0da 100644 --- a/contrib/gitian/gitian-osx.yml +++ b/contrib/gitian/gitian-osx.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-osx-0.18" +name: "monero-osx-0.14" enable_cache: true suites: - "bionic" diff --git a/contrib/gitian/gitian-win.yml b/contrib/gitian/gitian-win.yml index 74c757c11..fef5567f9 100644 --- a/contrib/gitian/gitian-win.yml +++ b/contrib/gitian/gitian-win.yml @@ -1,5 +1,5 @@ --- -name: "bitcoin-win-0.18" +name: "monero-win-0.14" enable_cache: true suites: - "bionic"