From 2e4d9eeec5e7cb349d627fb07b3672a2e84976df Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 13 May 2015 15:26:44 +0200 Subject: [PATCH 01/34] sys/boot/common/newvers.sh: Allow to overwrite the date to make boot loader binaries reproducible --- sys/boot/common/newvers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/common/newvers.sh b/sys/boot/common/newvers.sh index cd7422f..8c5e9d6 100755 --- a/sys/boot/common/newvers.sh +++ b/sys/boot/common/newvers.sh @@ -36,7 +36,7 @@ tempfile=$(mktemp tmp.XXXXXX) || exit trap "rm -f $tempfile" EXIT INT TERM LC_ALL=C; export LC_ALL -u=${USER-root} h=${HOSTNAME-`hostname`} t=`date` +u=${USER-root} h=${HOSTNAME-`hostname`} t=${DATE-`date`} #r=`head -n 6 $1 | tail -n 1 | awk -F: ' { print $1 } '` r=`awk -F: ' /^[0-9]\.[0-9]+:/ { print $1; exit }' $1` -- 2.6.3 From 0c94fcd63e94aa86716e9bc6ff392874779de47f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 13 May 2015 15:36:48 +0200 Subject: [PATCH 02/34] sys/conf/newvers.sh: Allow to overwrite the build date embedded into the kernel This is a required step to get reproducible builds. --- sys/conf/newvers.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index e14e377..388b4b7 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -98,7 +98,12 @@ fi touch version v=`cat version` u=${USER:-root} d=`pwd` h=${HOSTNAME:-`hostname`} -if [ -n "$SOURCE_DATE_EPOCH" ]; then +if [ -n "${DATE}" ]; then + # SOURCE_DATE_EPOCH was added upstream in r291691 + # but the ElectroBSD build goo is still setting DATE + # which has a different format. + t=${DATE} +elif [ -n "$SOURCE_DATE_EPOCH" ]; then if ! t=`date -r $SOURCE_DATE_EPOCH 2>/dev/null`; then echo "Invalid SOURCE_DATE_EPOCH" >&2 exit 1 -- 2.6.3 From abff7b9a3260e628191d8b0e2ae52f4893d125a3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 18 May 2015 19:10:44 +0200 Subject: [PATCH 03/34] amd64/make-memstick.sh: Create more 'predictable' images According to the mkimg man page, the "-y option is used for testing purposes only and is not to be used in production", but it feels good to be a gangsta. --- release/amd64/make-memstick.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 6c289e0..387d6eb 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -36,6 +36,6 @@ if [ $? -ne 0 ]; then fi rm ${1}/etc/fstab -mkimg -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} +mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} rm ${2}.part -- 2.6.3 From ef4ff9a7ae9d0ccc3a7bf4853740714b0d4278be Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 18 May 2015 19:17:14 +0200 Subject: [PATCH 04/34] release/Makefile: Don't create matroshka src tarballs that contain other tarballs ... if DESTDIR isn't set to a reasonable value. While at it, exclude *.orig files as well. --- release/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/Makefile b/release/Makefile index 5de65fc..fa3707e 100644 --- a/release/Makefile +++ b/release/Makefile @@ -145,7 +145,9 @@ src.txz: mkdir -p ${DISTDIR}/usr ln -fs ${WORLDDIR} ${DISTDIR}/usr/src cd ${DISTDIR} && tar cLvf - --exclude .svn --exclude .zfs \ - --exclude .git --exclude @ --exclude usr/src/release/dist usr/src | \ + --exclude .git --exclude @ --exclude usr/src/release/dist \ + --exclude "usr/src/release/*.txz" \ + --exclude "usr/src/release/*.orig" usr/src | \ ${XZ_CMD} > ${.OBJDIR}/src.txz ports.txz: -- 2.6.3 From 5609f5ed2d8b898d051935701bdcfe0fd3acd399 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 11 May 2015 18:45:24 +0200 Subject: [PATCH 05/34] release: Use a hack to recreate dist tarballs with reproducible timestamps --- release/Makefile | 3 ++ release/scripts/tar-time-reset.sh | 64 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100755 release/scripts/tar-time-reset.sh diff --git a/release/Makefile b/release/Makefile index fa3707e..22a04f3 100644 --- a/release/Makefile +++ b/release/Makefile @@ -265,6 +265,9 @@ mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} + for tarball in *.txz; do \ + sh ${.CURDIR}/scripts/tar-time-reset.sh $${tarball}; \ + done sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST touch ${.TARGET} diff --git a/release/scripts/tar-time-reset.sh b/release/scripts/tar-time-reset.sh new file mode 100755 index 0000000..8186fd8 --- /dev/null +++ b/release/scripts/tar-time-reset.sh @@ -0,0 +1,64 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## +# +# This script resets the timestamps in a given tarfile to hopefully make +# it reproducible. This is a rather wasteful approach, but works for now. +# +# A better solution would be to patch bsdtar to optionally use a fixed +# time (without having to clown around with mtree specs). +# +########################################################################## + +main() { + local tarfile="${1}" \ + tempdir mtree_spec + + if [ $# -ne 1 ]; then + echo "$0 /path/to/tarfile" + exit 1 + fi + + if [ -z "${tarfile}" ]; then + echo "No tar file given" + return 1 + fi + tarfile="$(realpath "$tarfile")" + + tempdir=$(mktemp -d) || return 1 + mtree_spec=$(mktemp) || return 1 + + echo "Extracting tarfile ${tarfile}" + (cd "${tempdir}" && tar xvf "${tarfile}") || return 1 + + echo "Ditching original tarfile ${tarfile}" + rm "${tarfile}" + + echo "Creating mtree spec in ${mtree_spec}" + (cd "${tempdir}" && mtree -L -c -k time) | \ + sed "s@time=.*@time=${EPOCH_DATE-0}.000000000@" > "${mtree_spec}" + + echo "Creating tarfile ${tarfile}" + (cd "${tempdir}" && tar acLvf "${tarfile}" @"${mtree_spec}") || return 1 + + echo "Ditching ${tempdir}" + rm -r "${tempdir}" || return 1 + echo "Ditching ${mtree_spec}" + rm "${mtree_spec}" || return 1 +} + +main "${@}" -- 2.6.3 From d722d3ef44ab5004378b79bc8184ccccf560df09 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Tue, 19 May 2015 16:06:01 +0200 Subject: [PATCH 06/34] release/amd64/make-memstick.sh: Use reproducible timestamps for the makefs image --- release/amd64/make-memstick.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 387d6eb..534ec45 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -29,12 +29,14 @@ if [ -e ${2} ]; then fi echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab -makefs -B little -o label=FreeBSD_Install ${2}.part ${1} +mtree -c -k time -p "${1}" | sed "s@time=.*@time=${EPOCH_DATE-0}.000000000@" > "${2}.mtree" +makefs -B little -o label=FreeBSD_Install -F "${2}.mtree" ${2}.part ${1} if [ $? -ne 0 ]; then echo "makefs failed" exit 1 fi rm ${1}/etc/fstab +rm "${2}.mtree" mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} rm ${2}.part -- 2.6.3 From fe6c5317dba2548c720699fac30b6baa0cefcae4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 13 May 2015 14:25:17 +0200 Subject: [PATCH 07/34] Remove build timestamps from ntp* binaries again. r195626 --- contrib/ntp/scripts/build/mkver.in | 2 -- 1 file changed, 2 deletions(-) mode change 100644 => 100755 contrib/ntp/scripts/build/mkver.in diff --git a/contrib/ntp/scripts/build/mkver.in b/contrib/ntp/scripts/build/mkver.in old mode 100644 new mode 100755 index 3aef1c8..badd2cd --- a/contrib/ntp/scripts/build/mkver.in +++ b/contrib/ntp/scripts/build/mkver.in @@ -15,8 +15,6 @@ case "@VER_SUFFIX@" in *) ConfStr="${ConfStr}-@VER_SUFFIX@" ;; esac -ConfStr="$ConfStr `LC_TIME=C TZ=UTC date`" - if [ ! -f .version ]; then echo 0 > .version fi -- 2.6.3 From 20e003e52eb6b3c40773cacad0d0bef0a01f7d9b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 20 May 2015 12:43:25 +0200 Subject: [PATCH 08/34] usr.sbin/ntp: Allow to set MKREPRO_DATE and MKREPRO_TIME to get reproducible builds --- usr.sbin/ntp/libntp/Makefile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/usr.sbin/ntp/libntp/Makefile b/usr.sbin/ntp/libntp/Makefile index 1e48483..95d3466 100644 --- a/usr.sbin/ntp/libntp/Makefile +++ b/usr.sbin/ntp/libntp/Makefile @@ -81,6 +81,13 @@ CFLAGS+= -I${.CURDIR}/../../../contrib/ntp/include \ CFLAGS+= -DHAVE_BSD_NICE -DHAVE_STDINT_H +.if defined(MKREPRO_DATE) +CFLAGS+= -DMKREPRO_DATE="\"${MKREPRO_DATE}\"" +.endif +.if defined(MKREPRO_TIME) +CFLAGS+= -DMKREPRO_TIME="\"${MKREPRO_TIME}\"" +.endif + CLEANFILES+= .version version.c version.c: -- 2.6.3 From d8fff396dfb14924b0eeb7db0121dc33bd0cae49 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 22 May 2015 11:33:50 +0200 Subject: [PATCH 09/34] release: Allow to build the mtree spec for mergemaster reproducible This relies on NetBSD mtree which has been the default for a while now. --- release/scripts/mm-mtree.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/release/scripts/mm-mtree.sh b/release/scripts/mm-mtree.sh index 4499c10..d5229be 100755 --- a/release/scripts/mm-mtree.sh +++ b/release/scripts/mm-mtree.sh @@ -146,8 +146,11 @@ find ${TEMPROOT} -type f -size 0 -delete 2>/dev/null find -d ${TEMPROOT} -type d -empty -delete 2>/dev/null # Build the mtree database in a temporary location. +# The second mtree call is used to get a reproducible result +# without embedded hostname, user name and creation timestamp. MTREENEW=`mktemp -t mergemaster.mtree` -mtree -ci -p ${TEMPROOT} -k size,md5digest > ${MTREENEW} 2>/dev/null +mtree -ci -p ${TEMPROOT} -k size,md5digest 2>/dev/null | \ + mtree -C -k all > ${MTREENEW} if [ -s "${MTREENEW}" ]; then echo "*** Saving mtree database for future upgrades" -- 2.6.3 From c63c8a6122ce647d840f139304f668a5e1239be7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 14 May 2015 12:07:25 +0200 Subject: [PATCH 10/34] Make reproducing builds more convenient ... by setting the various variables based on the environment variable REPRO_SEED. --- Makefile.inc1 | 14 +++++++--- release/Makefile | 2 ++ share/mk/src.reproducible-build.mk | 54 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 share/mk/src.reproducible-build.mk diff --git a/Makefile.inc1 b/Makefile.inc1 index 854ef6b..aefbfdd 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -48,6 +48,8 @@ .error "Both TARGET and TARGET_ARCH must be defined." .endif +.include "share/mk/src.reproducible-build.mk" + # Cross toolchain changes must be in effect before bsd.compiler.mk # so that gets the right CC, and pass CROSS_TOOLCHAIN to submakes. .if defined(CROSS_TOOLCHAIN) @@ -754,17 +756,23 @@ WMAKE_TGTS+= build32 buildworld: buildworld_prologue ${WMAKE_TGTS} buildworld_epilogue .ORDER: buildworld_prologue ${WMAKE_TGTS} buildworld_epilogue -buildworld_prologue: +buildworld_prologue: reproducible_build_hint @echo "--------------------------------------------------------------" @echo ">>> World build started on `LC_ALL=C date`" @echo "--------------------------------------------------------------" -buildworld_epilogue: +buildworld_epilogue: reproducible_build_hint @echo @echo "--------------------------------------------------------------" @echo ">>> World build completed on `LC_ALL=C date`" @echo "--------------------------------------------------------------" +reproducible_build_hint: + @echo "--------------------------------------------------------------" + @echo ">>> To reproduce this build:" + @echo ">>> export REPRO_SEED=$${REPRO_SEED}" + @echo "--------------------------------------------------------------" + # # We need to have this as a target because the indirection between Makefile # and Makefile.inc1 causes the correct PATH to be used, rather than a @@ -1129,7 +1137,7 @@ ${WMAKE_TGTS:N_worldtmp:Nbuild32} ${.ALLTARGETS:M_*:N_worldtmp}: .MAKE .PHONY # # Builds all kernels defined by BUILDKERNELS. # -buildkernel: .MAKE .PHONY +buildkernel: .MAKE .PHONY reproducible_build_hint .if empty(BUILDKERNELS:Ndummy) @echo "ERROR: Missing kernel configuration file(s) (${KERNCONF})."; \ false diff --git a/release/Makefile b/release/Makefile index 22a04f3..5a9be64 100644 --- a/release/Makefile +++ b/release/Makefile @@ -36,6 +36,8 @@ # TARGET/TARGET_ARCH: architecture of built release # +.include "../share/mk/src.reproducible-build.mk" + WORLDDIR?= ${.CURDIR}/.. PORTSDIR?= /usr/ports DOCDIR?= /usr/doc diff --git a/share/mk/src.reproducible-build.mk b/share/mk/src.reproducible-build.mk new file mode 100644 index 0000000..dc4f4df --- /dev/null +++ b/share/mk/src.reproducible-build.mk @@ -0,0 +1,54 @@ +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## +# +# Make the build reproducible by exporting a bunch of variables, +# potentionally using an already-set REPRO_SEED as input. +# +# The variable names are somewhat stupid, mostly because we +# are using existing ones. +# +########################################################################## + +TZ= "UTC" +.export TZ + +.if ! defined(REPRO_SEED) +# XXX: Currently we don't add the kernel version number to the repro +# seed because we expect a clean object tree in which case it +# will reproducible be 0. If the object tree of a the +# build-to-reproduce was actually unclean, KERNEL_VERSION_NUMBER +# has to be set to a matching value. +REPRO_SEED!= echo $$(id -un):$$(hostname):$$(date +%s) +.export REPRO_SEED +.else +.if ! defined(KERNEL_VERSION_NUMBER) +KERNEL_VERSION_NUMBER=0 +.export KERNEL_VERSION_NUMBER +.endif +.endif + +USER!= echo "${REPRO_SEED}" | /usr/bin/cut -d : -f 1 +HOSTNAME!= echo "${REPRO_SEED}" | /usr/bin/cut -d : -f 2 +EPOCH_DATE!= echo "${REPRO_SEED}" | /usr/bin/cut -d : -f 3 + +DATE!= date -r ${EPOCH_DATE} +# These two probably are no longer necessary after r285701 +MKREPRO_DATE!= date -r ${EPOCH_DATE} +"%b %d %Y" +MKREPRO_TIME!= date -r ${EPOCH_DATE} +%H:%M:%S + +.for v in REPRO_SEED USER HOSTNAME EPOCH_DATE DATE MKREPRO_DATE MKREPRO_TIME +.export $v +.endfor -- 2.6.3 From 7fd40e024c396bba21b854027b3d4484afff5e9f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 23 May 2015 20:13:18 +0200 Subject: [PATCH 11/34] contrib/groff/mdate.sh: Use ${EPOCH_DATE} when set While this is silly, using the time the man pages have been last checked out from the VCS is silly as well, so this commit doesn't increase the total amount of sillyness in the system. Also we are talking about GNU roff here, so hopefully this stuff will be garbage-collected in the near future. --- contrib/groff/mdate.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/groff/mdate.sh b/contrib/groff/mdate.sh index 4a26e6e..615abc7 100755 --- a/contrib/groff/mdate.sh +++ b/contrib/groff/mdate.sh @@ -1,12 +1,17 @@ #! /bin/sh # Print the modification date of $1 `nicely'. +# If ${EPOCH_DATE} is set, it will be used instead. # Don't want foreign dates. LANGUAGE= LC_ALL=C; export LC_ALL +if [ -n "${EPOCH_DATE}" ]; then + date -r "${EPOCH_DATE}" +"%d %B %Y" + exit 0 +fi (date; if ls -L /dev/null 1>/dev/null 2>&1; then ls -L -l $1; else ls -l $1; fi -- 2.6.3 From e72019c57006c987dc36e5553a2bc4a83b6e7b10 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sun, 24 May 2015 18:01:31 +0200 Subject: [PATCH 12/34] Fake modification time smarter for everything but EXTRA_PACKAGES While at it, remove duplicated slashes in the METALOG as they result in missing files. --- Makefile.inc1 | 17 ++++++++++++++++- release/Makefile | 6 +++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index aefbfdd..422bb7a 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -985,6 +985,13 @@ distributeworld installworld: _installcheck_world find ${DESTDIR}/${DISTDIR}/${dist} -mindepth 1 -empty -delete .endfor .if defined(NO_ROOT) + @# Post process METALOG: add fake timestamps and, if necessary, + @# remove duplicated slashes which can occur if DISTDIR is undefined. + @# Keeping them results in missing files in the distribution tarballs. +.if defined(EPOCH_DATE) + sed -E -e 's@time=[0-9]+.0@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ + -e 's@//@/@' -i '.bak' ${METALOG} +.endif .for dist in base ${EXTRA_DISTRIBUTIONS} @# For each file that exists in this dist, print the corresponding @# line from the METALOG. This relies on the fact that @@ -1012,7 +1019,7 @@ packageworld: .for dist in base ${EXTRA_DISTRIBUTIONS} .if defined(NO_ROOT) ${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \ - tar cvf - --exclude usr/lib/debug \ + tar cvf - \ @${DESTDIR}/${DISTDIR}/${dist}.meta | \ ${XZ_CMD} > ${PACKAGEDIR}/${dist}.txz .else @@ -1267,12 +1274,20 @@ distributekernel distributekernel.debug: packagekernel: .if defined(NO_ROOT) .if !defined(NO_INSTALLKERNEL) +.if defined(EPOCH_DATE) + sed -E -e 's@time=[0-9]+.0@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ + -i '.bak' ${DESTDIR}/${DISTDIR}/kernel.meta +.endif cd ${DESTDIR}/${DISTDIR}/kernel; \ tar cvf - @${DESTDIR}/${DISTDIR}/kernel.meta | \ ${XZ_CMD} > ${PACKAGEDIR}/kernel.txz .endif .if ${BUILDKERNELS:[#]} > 1 .for _kernel in ${BUILDKERNELS:[2..-1]} +.if defined(EPOCH_DATE) + sed -E -e 's@time=[0-9]+.0@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ + -i '.bak' ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta +.endif cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \ tar cvf - @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta | \ ${XZ_CMD} > ${PACKAGEDIR}/kernel.${_kernel}.txz diff --git a/release/Makefile b/release/Makefile index 5a9be64..d44f375 100644 --- a/release/Makefile +++ b/release/Makefile @@ -56,6 +56,10 @@ TARGET_ARCH= ${TARGET} IMAKE= ${MAKE} TARGET_ARCH=${TARGET_ARCH} TARGET=${TARGET} DISTDIR= dist +# Enable mtree spec usage so we can fake the modification time. +NO_ROOT=1 +.export NO_ROOT + # Define OSRELEASE by using newvars.sh .if !defined(OSRELEASE) || empty(OSRELEASE) .for _V in TYPE BRANCH REVISION @@ -267,7 +271,7 @@ mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} - for tarball in *.txz; do \ + for tarball in ${EXTRA_PACKAGES}; do \ sh ${.CURDIR}/scripts/tar-time-reset.sh $${tarball}; \ done sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST -- 2.6.3 From e6a234f330acf1c6dd3780f3e7fd675bddd94508 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 25 May 2015 10:27:48 +0200 Subject: [PATCH 13/34] Add image-checksum.sh ... which calculates a checksum of the reproducible parts of an memstick image. --- release/scripts/image-checksum.sh | 156 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100755 release/scripts/image-checksum.sh diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh new file mode 100755 index 0000000..6f79a81 --- /dev/null +++ b/release/scripts/image-checksum.sh @@ -0,0 +1,156 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## +# +# image-checksum.sh /path/to/memstick.img +# +# Unfortunately the memstick target currently does not create reproducible +# ElectroBSD images due to unreproducible differences in the file system +# layer of the data partition. +# +# To be able to (sort of) compare memstick images anyway, this script +# produces a "partial image checksum" that is based on the partition layout, +# the checksum of the boot code partition and an mtree spec of the data +# partition which includes checksums, sizes and timestamps for all the +# files. +# +# A memstick image whose "partial checksum" matches the one of another +# image can be totally considered to be nearly as trustworthy. Obviously +# that's a somewhat worthless property, it is thus recommended that you +# rebuild the potentionally malicious image using a trusted operating +# system first. After you've done this, potentionally malicious differences +# in the unchecked parts should be gone. +# +# Just kidding, image-checksum.sh is only intended to regression-test +# the ElectroBSD build system. +# +# Also note that this script relies on non-standardized output of other +# tools which might occasionally change. To be able to reproduce partial +# image checksums you thus need a userland that is close enough to the +# one that was used to create the original version. +# +########################################################################## + +UFS_PARTITION=p2 +EXPECTED_PARTITIONS=2 +MOUNTPOINT=/mnt +VERBOSE=0 +MTREE_KEYWORDS=size,time,uid,gid,sha256 + +verbose_log() { + local message="$*" + if [ "${VERBOSE}" = 0 ]; then + return + fi + echo "${message}" +} + +create_mtree_spec_file() { + local md_unit spec_file + + md_unit="${1}" + spec_file="${2}" + + verbose_log "Mounting /dev/md${md_unit}${UFS_PARTITION} at ${MOUNTPOINT}" + mount -o ro "/dev/md${md_unit}${UFS_PARTITION}" "${MOUNTPOINT}" || return 1 + + verbose_log "Running mtree, saving spec in ${spec_file}" + mtree -c -k "${MTREE_KEYWORDS}" -p "${MOUNTPOINT}" | mtree -C -k all > "${spec_file}" || return 1 + + verbose_log "Unmounting ${MOUNTPOINT} ..." + umount "${MOUNTPOINT}" || return 1 +} + +partition_count_acceptable() { + local md_unit="${1}" + + # Verify that there are exactly two partitions present + partitions=$(gpart show -r -p "md${md_unit}" | grep -c "md${md_unit}"p) + if [ "${partitions}" != "${EXPECTED_PARTITIONS}" ]; then + echo "Invalid number of partitions: ${partitions}" + return 1; + fi +} + +main() { + local image_file \ + args md_unit spec_file gpart_file + + args=$(getopt v $*) + if [ $? -ne 0 ]; then + echo 'You are doing it wrong: Invalid flag specified' + exit 2 + fi + set -- ${args} + while true; do + case "$1" in + -v) + VERBOSE=1 + shift + ;; + --) + shift; break + ;; + esac + done + + image_file=${1} + if [ -z "${image_file}" ]; then + echo "No image file provided" + return 1 + fi + spec_file="${image_file}.mtree" + if [ -f "${spec_file}" ]; then + echo "Spec file ${spec_file} already exists" + return 1 + fi + gpart_file="${image_file}.gpart" + if [ -f "${spec_file}" ]; then + echo "gpart file ${gpart_file} already exists" + return 1 + fi + + md_unit=$(mdconfig -o readonly -n -f "${image_file}") + if [ $? != 0 ]; then + return 1 + fi + + partition_count_acceptable "${md_unit}" || return 1 + + if [ ! -f "${spec_file}" ]; then + create_mtree_spec_file "${md_unit}" "${spec_file}" || return 1 + fi + if [ ! -f "${gpart_file}" ]; then + gpart list "md${md_unit}" | sed -E -e "s@(: md)${md_unit}@\1X@" > "${gpart_file}" + fi + + gpart_checksum=$(sha256 -q "${gpart_file}") + verbose_log "gpart checksum: ${gpart_checksum}" + + mdconfig -d -u "${md_unit}" || return 1 + + bootcode_checksum=$(dd if=/dev/md${md_unit}p1 2>/dev/null | sha256) + verbose_log "Boot code checksum: ${bootcode_checksum}" + mtree_checksum=$(sha256 -q "${spec_file}") + verbose_log "mtree checksum: ${mtree_checksum}" + + weak_image_checksum=$(echo "${gpart_checksum} ${bootcode_checksum} ${mtree_checksum}" | sha256) + echo "Partial image checksum: ${weak_image_checksum}" + +} + +main "${@}" -- 2.6.3 From 9dd13e69a781b9e1ec45a73bcbebed4890d45f9e Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 19:54:10 +0200 Subject: [PATCH 14/34] image-checksum: Add -r flag to reuse cache files --- release/scripts/image-checksum.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh index 6f79a81..1940921 100755 --- a/release/scripts/image-checksum.sh +++ b/release/scripts/image-checksum.sh @@ -50,6 +50,7 @@ EXPECTED_PARTITIONS=2 MOUNTPOINT=/mnt VERBOSE=0 MTREE_KEYWORDS=size,time,uid,gid,sha256 +REUSE_EXISTING_CACHE_FILES=false verbose_log() { local message="$*" @@ -90,7 +91,7 @@ main() { local image_file \ args md_unit spec_file gpart_file - args=$(getopt v $*) + args=$(getopt rv $*) if [ $? -ne 0 ]; then echo 'You are doing it wrong: Invalid flag specified' exit 2 @@ -98,6 +99,10 @@ main() { set -- ${args} while true; do case "$1" in + -r) + REUSE_EXISTING_CACHE_FILES=true + shift + ;; -v) VERBOSE=1 shift @@ -116,12 +121,12 @@ main() { spec_file="${image_file}.mtree" if [ -f "${spec_file}" ]; then echo "Spec file ${spec_file} already exists" - return 1 + ${REUSE_EXISTING_CACHE_FILES} || return 1 fi gpart_file="${image_file}.gpart" if [ -f "${spec_file}" ]; then echo "gpart file ${gpart_file} already exists" - return 1 + ${REUSE_EXISTING_CACHE_FILES} || return 1 fi md_unit=$(mdconfig -o readonly -n -f "${image_file}") -- 2.6.3 From d17befc8a5a741b565156dadd53dac68de5d7d49 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 3 Jul 2015 20:20:01 +0200 Subject: [PATCH 15/34] image-checksum.sh: Allow to overwrite the mtree flags --- release/scripts/image-checksum.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh index 1940921..cc31e90 100755 --- a/release/scripts/image-checksum.sh +++ b/release/scripts/image-checksum.sh @@ -91,7 +91,7 @@ main() { local image_file \ args md_unit spec_file gpart_file - args=$(getopt rv $*) + args=$(getopt m:rv $*) if [ $? -ne 0 ]; then echo 'You are doing it wrong: Invalid flag specified' exit 2 @@ -99,6 +99,11 @@ main() { set -- ${args} while true; do case "$1" in + -m) + shift + MTREE_KEYWORDS="${1}" + shift + ;; -r) REUSE_EXISTING_CACHE_FILES=true shift -- 2.6.3 From 8b179345f29041e21fcd26164e081e90aac201dc Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 31 Aug 2015 10:51:16 +0200 Subject: [PATCH 16/34] release/scripts/image-checksum.sh: Allow to checksum multiple images at once --- release/scripts/image-checksum.sh | 79 ++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/release/scripts/image-checksum.sh b/release/scripts/image-checksum.sh index cc31e90..30ac62a 100755 --- a/release/scripts/image-checksum.sh +++ b/release/scripts/image-checksum.sh @@ -87,42 +87,11 @@ partition_count_acceptable() { fi } -main() { +generate_partial_image_checksum() { local image_file \ - args md_unit spec_file gpart_file - - args=$(getopt m:rv $*) - if [ $? -ne 0 ]; then - echo 'You are doing it wrong: Invalid flag specified' - exit 2 - fi - set -- ${args} - while true; do - case "$1" in - -m) - shift - MTREE_KEYWORDS="${1}" - shift - ;; - -r) - REUSE_EXISTING_CACHE_FILES=true - shift - ;; - -v) - VERBOSE=1 - shift - ;; - --) - shift; break - ;; - esac - done + md_unit spec_file gpart_file - image_file=${1} - if [ -z "${image_file}" ]; then - echo "No image file provided" - return 1 - fi + image_file="${1}" spec_file="${image_file}.mtree" if [ -f "${spec_file}" ]; then echo "Spec file ${spec_file} already exists" @@ -159,8 +128,48 @@ main() { verbose_log "mtree checksum: ${mtree_checksum}" weak_image_checksum=$(echo "${gpart_checksum} ${bootcode_checksum} ${mtree_checksum}" | sha256) - echo "Partial image checksum: ${weak_image_checksum}" + echo "Partial image checksum for ${image_file}: ${weak_image_checksum}" +} +main() { + local image_file \ + args + + args=$(getopt m:rv $*) + if [ $? -ne 0 ]; then + echo 'You are doing it wrong: Invalid flag specified' + exit 2 + fi + set -- ${args} + while true; do + case "$1" in + -m) + shift + MTREE_KEYWORDS="${1}" + shift + ;; + -r) + REUSE_EXISTING_CACHE_FILES=true + shift + ;; + -v) + VERBOSE=1 + shift + ;; + --) + shift; break + ;; + esac + done + + if [ -z "${1}" ]; then + echo "No image file provided" + return 1 + fi + + for image_file in "${@}"; do + generate_partial_image_checksum "${image_file}" || return 1 + done } main "${@}" -- 2.6.3 From 835ffe8ea1fd0c9c3f19fc0c6c63f314bbc4491f Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 28 May 2015 15:04:48 +0200 Subject: [PATCH 17/34] sys/conf/newvers.sh: Allow to overwrite the kernel version ... as the object directory may be dirty. --- sys/conf/newvers.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 388b4b7..29bf618 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -97,7 +97,8 @@ then fi touch version -v=`cat version` u=${USER:-root} d=`pwd` h=${HOSTNAME:-`hostname`} +v=${KERNEL_VERSION_NUMBER:-`cat version`} +u=${USER:-root} d=`pwd` h=${HOSTNAME:-`hostname`} if [ -n "${DATE}" ]; then # SOURCE_DATE_EPOCH was added upstream in r291691 # but the ElectroBSD build goo is still setting DATE @@ -244,4 +245,6 @@ int osreldate = ${RELDATE}; char kern_ident[] = "${i}"; EOF -echo $((v + 1)) > version +if [ -z "${KERNEL_VERSION_NUMBER}" ]; then + echo $((v + 1)) > version +fi -- 2.6.3 From 7aa6c512ee3ac263869920c067d86b0353696a00 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 30 May 2015 14:38:48 +0200 Subject: [PATCH 18/34] sys: Do not embed compiler version in kernel binary as it makes reproducing the binary with a different compiler more complicated. --- sys/conf/newvers.sh | 2 -- sys/kern/init_main.c | 1 - sys/kern/kern_mib.c | 3 --- sys/sys/systm.h | 1 - 4 files changed, 7 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 29bf618..84bb28a 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -113,7 +113,6 @@ else t=`date` fi i=`${MAKE:-make} -V KERN_IDENT` -compiler_v=$($(${MAKE:-make} -V CC) -v 2>&1 | grep -w 'version') for dir in /usr/bin /usr/local/bin; do if [ ! -z "${svnversion}" ] ; then @@ -238,7 +237,6 @@ $COPYRIGHT char sccs[sizeof(SCCSSTR) > 128 ? sizeof(SCCSSTR) : 128] = SCCSSTR; char version[sizeof(VERSTR) > 256 ? sizeof(VERSTR) : 256] = VERSTR; -char compiler_version[] = "${compiler_v}"; char ostype[] = "${TYPE}"; char osrelease[sizeof(RELSTR) > 32 ? sizeof(RELSTR) : 32] = RELSTR; int osreldate = ${RELDATE}; diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index f0cd3c8..cb3acae 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -341,7 +341,6 @@ print_version(void *data __unused) while (len > 0 && version[len - 1] == '\n') len--; printf("%.*s %s\n", len, version, machine); - printf("%s\n", compiler_version); } SYSINIT(announce, SI_SUB_COPYRIGHT, SI_ORDER_FIRST, print_caddr_t, diff --git a/sys/kern/kern_mib.c b/sys/kern/kern_mib.c index 2ee260d..30f8d73 100644 --- a/sys/kern/kern_mib.c +++ b/sys/kern/kern_mib.c @@ -97,9 +97,6 @@ SYSCTL_INT(_kern, KERN_OSREV, osrevision, CTLFLAG_RD|CTLFLAG_CAPRD, SYSCTL_STRING(_kern, KERN_VERSION, version, CTLFLAG_RD|CTLFLAG_MPSAFE, version, 0, "Kernel version"); -SYSCTL_STRING(_kern, OID_AUTO, compiler_version, CTLFLAG_RD|CTLFLAG_MPSAFE, - compiler_version, 0, "Version of compiler used to compile kernel"); - SYSCTL_STRING(_kern, KERN_OSTYPE, ostype, CTLFLAG_RD|CTLFLAG_MPSAFE| CTLFLAG_CAPRD, ostype, 0, "Operating system type"); diff --git a/sys/sys/systm.h b/sys/sys/systm.h index 026a03c..7cd7abc 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -50,7 +50,6 @@ extern int suspend_blocked; /* block suspend due to pending shutdown */ extern int rebooting; /* kern_reboot() has been called. */ extern const char *panicstr; /* panic message */ extern char version[]; /* system version */ -extern char compiler_version[]; /* compiler version */ extern char copyright[]; /* system copyright */ extern int kstack_pages; /* number of kernel stack pages */ -- 2.6.3 From bb61e9e9122c242415e0790a35e00c07083217e7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Sat, 30 May 2015 14:42:43 +0200 Subject: [PATCH 19/34] sys/conf/newvers.sh: Ditch support for p4 (non-free) and hg (not relevant for ElectroBSD) --- sys/conf/newvers.sh | 46 ++-------------------------------------------- 1 file changed, 2 insertions(+), 44 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 84bb28a..33a9e33 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -139,11 +139,6 @@ if [ -z "${svnversion}" ] && [ -x /usr/bin/svnliteversion ] ; then fi fi -for dir in /usr/bin /usr/local/bin; do - if [ -x "${dir}/p4" ] && [ -z ${p4_cmd} ] ; then - p4_cmd=${dir}/p4 - fi -done if [ -d "${SYSDIR}/../.git" ] ; then for dir in /usr/bin /usr/local/bin; do if [ -x "${dir}/git" ] ; then @@ -153,15 +148,6 @@ if [ -d "${SYSDIR}/../.git" ] ; then done fi -if [ -d "${SYSDIR}/../.hg" ] ; then - for dir in /usr/bin /usr/local/bin; do - if [ -x "${dir}/hg" ] ; then - hg_cmd="${dir}/hg -R ${SYSDIR}/.." - break - fi - done -fi - if [ -n "$svnversion" ] ; then svn=`cd ${SYSDIR} && $svnversion 2>/dev/null` case "$svn" in @@ -201,38 +187,10 @@ if [ -n "$git_cmd" ] ; then # fi fi -if [ -n "$p4_cmd" ] ; then - p4version=`cd ${SYSDIR} && $p4_cmd changes -m1 "./...#have" 2>&1 | \ - awk '{ print $2 }'` - case "$p4version" in - [0-9]*) - p4version=" ${p4version}" - p4opened=`cd ${SYSDIR} && $p4_cmd opened ./... 2>&1` - case "$p4opened" in - File*) ;; - //*) p4version="${p4version}+edit" ;; - esac - ;; - *) unset p4version ;; - esac -fi - -if [ -n "$hg_cmd" ] ; then - hg=`$hg_cmd id 2>/dev/null` - svn=`$hg_cmd svn info 2>/dev/null | \ - awk -F': ' '/Revision/ { print $2 }'` - if [ -n "$svn" ] ; then - svn=" r${svn}" - fi - if [ -n "$hg" ] ; then - hg=" ${hg}" - fi -fi - cat << EOF > vers.c $COPYRIGHT -#define SCCSSTR "@(#)${VERSION} #${v}${svn}${git}${hg}${p4version}: ${t}" -#define VERSTR "${VERSION} #${v}${svn}${git}${hg}${p4version}: ${t}\\n ${u}@${h}:${d}\\n" +#define SCCSSTR "@(#)${VERSION} #${v}${svn}${git}: ${t}" +#define VERSTR "${VERSION} #${v}${svn}${git}: ${t}\\n ${u}@${h}:${d}\\n" #define RELSTR "${RELEASE}" char sccs[sizeof(SCCSSTR) > 128 ? sizeof(SCCSSTR) : 128] = SCCSSTR; -- 2.6.3 From af3406dc4a01c107968d115c1c36db3406a04854 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 3 Jun 2015 17:16:10 +0200 Subject: [PATCH 20/34] mandocdb: Normalize inodevs to hopefully get reproducible results --- contrib/mdocml/mandocdb.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/contrib/mdocml/mandocdb.c b/contrib/mdocml/mandocdb.c index fed11e9..b299386 100644 --- a/contrib/mdocml/mandocdb.c +++ b/contrib/mdocml/mandocdb.c @@ -915,6 +915,34 @@ filescan(const char *file) mlink_add(mlink, &st); } +/* + * Messes up inodevs in a reproducible way as long as + * the call order does not change. The implementation + * is silly and only used as proof of concept. + */ +#define HOPEFULLY_ENOUGH_FOR_EVERYBODY 5000 +static void +normalize_inodev(struct inodev *inodev) { + static size_t table[HOPEFULLY_ENOUGH_FOR_EVERYBODY]; + size_t key; + int i; + + key = inodev->st_ino + inodev->st_dev; + + for (i = 0; i < sizeof(table)/sizeof(table[0]); i++) { + if (table[i] == 0) { + /* New value, add to table*/ + table[i] = key; + } + if (table[i] == key) { + /* Use index as new value */ + inodev->st_ino = i; + inodev->st_dev = i; + return; + } + } +} + static void mlink_add(struct mlink *mlink, const struct stat *st) { @@ -945,6 +973,7 @@ mlink_add(struct mlink *mlink, const struct stat *st) memset(&inodev, 0, sizeof(inodev)); /* Clear padding. */ inodev.st_ino = st->st_ino; inodev.st_dev = st->st_dev; + normalize_inodev(&inodev); slot = ohash_lookup_memory(&mpages, (char *)&inodev, sizeof(struct inodev), inodev.st_ino); mpage = ohash_find(&mpages, slot); -- 2.6.3 From e84f8d8d2c957b157b73cb1f1a2d575e6edf44a4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 1 Jun 2015 13:56:05 +0200 Subject: [PATCH 21/34] release/Makefile: Build the src.txz only once and also fake source ownership ... after hours of blood, sweat and tears it finally (appears to) work. Keep release/scripts/tar-time-reset.sh around but update a comment to make it obvious that the script isn't used anymore --- release/Makefile | 27 +++++++++++++++++++-------- release/scripts/tar-time-reset.sh | 8 +++----- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/release/Makefile b/release/Makefile index d44f375..4169ae1 100644 --- a/release/Makefile +++ b/release/Makefile @@ -150,11 +150,25 @@ kernel.txz: src.txz: mkdir -p ${DISTDIR}/usr ln -fs ${WORLDDIR} ${DISTDIR}/usr/src - cd ${DISTDIR} && tar cLvf - --exclude .svn --exclude .zfs \ - --exclude .git --exclude @ --exclude usr/src/release/dist \ - --exclude "usr/src/release/*.txz" \ - --exclude "usr/src/release/*.orig" usr/src | \ - ${XZ_CMD} > ${.OBJDIR}/src.txz +# Create an mtree spec with faked timestamps so we get a reproducible +# tar file. We do not use tar for this because it's mtree generator +# appears to be buggy and exits with an memory allocation failure. +# +# It's important that the excluded paths start with "./", otherwise +# file locations are not recorded correctly, and, for example, +# usr/src/usr.sbin appears as usr/src/release/usr.sbin in the +# tar file. Only the shadow knows if that's a bug or a feature. + echo "./usr/src/.git" >${.OBJDIR}/mtree-exclude + echo "./usr/src/release/dist" >>${.OBJDIR}/mtree-exclude + echo "./usr/src/release/src.mtree" >>${.OBJDIR}/mtree-exclude + echo "./usr/src/release/mtree-exclude" >>${.OBJDIR}/mtree-exclude + cd ${DISTDIR} && mtree -c -L -k time -X ${.OBJDIR}/mtree-exclude | \ + sed -E -e 's@(time=)[0-9]+\.[0-9]+@\1${EPOCH_DATE}.0 uid=0 gid=0@' \ + > ${.OBJDIR}/src.mtree + rm ${.OBJDIR}/mtree-exclude + cd ${DISTDIR} && tar cLvf - @${.OBJDIR}/src.mtree \ + | ${XZ_CMD} > ${.OBJDIR}/src.txz + rm ${.OBJDIR}/src.mtree ports.txz: mkdir -p ${DISTDIR}/usr @@ -271,9 +285,6 @@ mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} - for tarball in ${EXTRA_PACKAGES}; do \ - sh ${.CURDIR}/scripts/tar-time-reset.sh $${tarball}; \ - done sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST touch ${.TARGET} diff --git a/release/scripts/tar-time-reset.sh b/release/scripts/tar-time-reset.sh index 8186fd8..f1b8434 100755 --- a/release/scripts/tar-time-reset.sh +++ b/release/scripts/tar-time-reset.sh @@ -17,11 +17,9 @@ ########################################################################## # # This script resets the timestamps in a given tarfile to hopefully make -# it reproducible. This is a rather wasteful approach, but works for now. -# -# A better solution would be to patch bsdtar to optionally use a fixed -# time (without having to clown around with mtree specs). -# +# it reproducible. As this is a rather wasteful approach the script is +# no longer used. It hasn't been removed yet as it may be useful for +# testing purposes. ########################################################################## main() { -- 2.6.3 From 66cb7266aae7c4320a69b4a86234312c9be6ff2b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 4 Jun 2015 12:18:16 +0200 Subject: [PATCH 22/34] release/Makefile: Reorder dependencies to reduce the chances that base or kernel parts end up in the src.txz. This is merely a workaround --- release/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/Makefile b/release/Makefile index 4169ae1..a077e5e 100644 --- a/release/Makefile +++ b/release/Makefile @@ -284,7 +284,7 @@ mini-memstick: mini-memstick.img mini-memstick.img: bootonly sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET} -packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES} +packagesystem: ${EXTRA_PACKAGES} base.txz kernel.txz sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST touch ${.TARGET} -- 2.6.3 From ffca42e34bf805ed46aa3b3a3486d1f04a235aac Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Mon, 17 Aug 2015 17:30:25 +0200 Subject: [PATCH 23/34] Add strip-freebsd.sh ... which suggests a bunch of stuff to delete from a vanilla FreeBSD checkout. In a previous life it was called free-freebsd.sh which sounds more awesome, but nowadays most of the suggested stuff for removal is actually free software that isn't relevant for ElectroBSD. --- release/scripts/strip-freebsd.sh | 168 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100755 release/scripts/strip-freebsd.sh diff --git a/release/scripts/strip-freebsd.sh b/release/scripts/strip-freebsd.sh new file mode 100755 index 0000000..a9088cf --- /dev/null +++ b/release/scripts/strip-freebsd.sh @@ -0,0 +1,168 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## + +# This script pretends to free a FreeBSD checkout from +# known-unfree files and other stuff that is not required +# by ElectroBSD. +# +# While it is pretty much guaranteed to ditch a bunch of files +# it doesn't work very thorougly and the resulting checkout is +# likely to still contain lots of non-free parts that haven't +# been discovered yet. + +get_snd_csa_files() { + find sys/dev/sound/pci -name "csa*" +} + +get_snd_ds1_files() { + find sys/dev/sound/pci -name "ds1*" +} + +# XXX: Does not actually seem to require binary blobs +#get_snd_maestro_files() { +# find sys/dev/sound/pci -name "maestro*" +#} + +# We can't simply remove the whole directory +# as a (free) header is required by bge. +get_bce_files() { + echo "sys/dev/bce/if_bcefw.h" \ + "sys/dev/bce/if_bce.c" +} + +get_usb_firmware_files() { + find sys/dev/usb/ -name "*fw*" +} + +# XXX: Misleading name, some of the files are merely tainted +# by non-free dependencies +get_unfree_files() { + find sys/ -name "*.uu" + get_snd_csa_files + get_snd_ds1_files + #get_snd_maestro_files + get_bce_files + get_usb_firmware_files +} + +get_files_to_ditch() { + get_unfree_files +} + +# These architectures are mainly unsupported by ElectroBSD +# due to lack of hardware for testing purposes. +# +# The source directories are mainly removed to shrink the +# source tarball and to reduce the number of files that +# should be audited for license and security issues. +get_unsupported_architectures() { + echo "arm arm64 mips pc98 powerpc sparc64" +} + +# These depend on or contain proprietary firmware that is included in sys/contrib/dev +get_tainted_sys_contrib_devs() { + echo "drm2 ipw iwi iwm iwn mwl npe otus ral rsu run uath urtwn wpi" +} + +# These require proprietary firmware that is included in sys/dev +# and may cause build failures without it. +get_tainted_sys_devs() { + # bce has already been taken care of by get_bce_files() above + echo "bxe ctau cx cxgb cxgbe ispfw qlxgbe" \ + "spibus it tw" +} + +get_unused_contrib_dirs() { + # XXX: gcc can't be deleted because parts of it are apparently + # required to build libc. This should be investiaged more thoroughly, + # hopefully it can be fixed. + echo "apr apr-util ipfilter ofed sendmail serf subversion tcsh" +} + +get_directories_to_ditch() { + local arch \ + dir arch_dir sys_contrib contrib_dir + + for dir in sys sys/boot; do + for arch in $(get_unsupported_architectures); do + potential_directory="${dir}/${arch}" + if [ -d "${potential_directory}" ]; then + echo "${potential_directory}" + fi + done + done + + for sys_contrib in ipfilter octeon-sdk; do + echo "sys/contrib/${sys_contrib}" + done + + for sys_contrib in $(get_tainted_sys_contrib_devs); do + echo "sys/contrib/dev/${sys_contrib}" + done + + for sys_dev in $(get_tainted_sys_devs); do + echo "sys/dev/${sys_dev}" + done + + for contrib_dir in $(get_unused_contrib_dirs); do + potential_directory="contrib/${contrib_dir}" + if [ -d "${potential_directory}" ]; then + echo "${potential_directory}" + fi + done +} + +purify_cwd() { + # There are no spaces in paths or file names. + files_to_ditch="$(get_files_to_ditch)" + for f in $files_to_ditch; do + [ -f "${f}" ] && echo "rm ${f}" + done + + dirs_to_ditch="$(get_directories_to_ditch)" + for d in $dirs_to_ditch; do + [ -d "${d}" ] && echo "rm -r ${d}" + done +} + +main() { + local src_dir \ + files_to_ditch dirs_to_ditch + + src_dir="${1}" + if [ -z "${src_dir}" ]; then + echo "No source directory given" + return 1 + fi + if [ ! -d "${src_dir}" ]; then + echo "No such directory: ${src_dir}" + return 1 + fi + # Make it less likely to operate on a directory + # that isn't actually a FreeBSD checkout + if [ ! -f "${src_dir}/COPYRIGHT" ]; then + echo "${src_dir} contains no COPYRIGHT file" + return 1 + fi + + cd "${src_dir}" || return 1 + + purify_cwd +} + +main "${@}" -- 2.6.3 From 3d91d48d7d745197629d6b5440fbd5a5ef6f116b Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 23 Sep 2015 13:53:34 +0200 Subject: [PATCH 24/34] Makefile.inc1: Mark two suspicious mtree spec modifications as such --- Makefile.inc1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile.inc1 b/Makefile.inc1 index 422bb7a..0e9bfff 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -1275,6 +1275,7 @@ packagekernel: .if defined(NO_ROOT) .if !defined(NO_INSTALLKERNEL) .if defined(EPOCH_DATE) +# XXX: Is this really necessary given that we already modify the METALOG itself? sed -E -e 's@time=[0-9]+.0@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ -i '.bak' ${DESTDIR}/${DISTDIR}/kernel.meta .endif @@ -1284,6 +1285,7 @@ packagekernel: .endif .if ${BUILDKERNELS:[#]} > 1 .for _kernel in ${BUILDKERNELS:[2..-1]} +# XXX: See XXX above .if defined(EPOCH_DATE) sed -E -e 's@time=[0-9]+.0@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ -i '.bak' ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta -- 2.6.3 From bc8bea4784ba4627451116dc714efc232a8856dc Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 23 Sep 2015 13:56:57 +0200 Subject: [PATCH 25/34] Makefile.inc: Try to make the release reproducible when not running as root --- Makefile.inc1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile.inc1 b/Makefile.inc1 index 0e9bfff..c2da7ec 100644 --- a/Makefile.inc1 +++ b/Makefile.inc1 @@ -989,7 +989,8 @@ distributeworld installworld: _installcheck_world @# remove duplicated slashes which can occur if DISTDIR is undefined. @# Keeping them results in missing files in the distribution tarballs. .if defined(EPOCH_DATE) - sed -E -e 's@time=[0-9]+.0@@' -e 's@(type=)@time=${EPOCH_DATE}.0 \1@' \ + sed -E -e 's@time=[0-9]+.0@@' \ + -e 's@(type=)@uid=0 gid=0 time=${EPOCH_DATE}.0 \1@' \ -e 's@//@/@' -i '.bak' ${METALOG} .endif .for dist in base ${EXTRA_DISTRIBUTIONS} -- 2.6.3 From b0dab625436a0357c62d32883a8a93e2c294e5e4 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Sep 2015 14:33:31 +0200 Subject: [PATCH 26/34] sys/conf/newvers.sh: Remove svn support (but keep 'git svn' support) --- sys/conf/newvers.sh | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 33a9e33..9880cd5 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -114,31 +114,6 @@ else fi i=`${MAKE:-make} -V KERN_IDENT` -for dir in /usr/bin /usr/local/bin; do - if [ ! -z "${svnversion}" ] ; then - break - fi - if [ -x "${dir}/svnversion" ] && [ -z ${svnversion} ] ; then - # Run svnversion from ${dir} on this script; if return code - # is not zero, the checkout might not be compatible with the - # svnversion being used. - ${dir}/svnversion $(realpath ${0}) >/dev/null 2>&1 - if [ $? -eq 0 ]; then - svnversion=${dir}/svnversion - break - fi - fi -done - -if [ -z "${svnversion}" ] && [ -x /usr/bin/svnliteversion ] ; then - /usr/bin/svnliteversion $(realpath ${0}) >/dev/null 2>&1 - if [ $? -eq 0 ]; then - svnversion=/usr/bin/svnliteversion - else - svnversion= - fi -fi - if [ -d "${SYSDIR}/../.git" ] ; then for dir in /usr/bin /usr/local/bin; do if [ -x "${dir}/git" ] ; then @@ -148,14 +123,6 @@ if [ -d "${SYSDIR}/../.git" ] ; then done fi -if [ -n "$svnversion" ] ; then - svn=`cd ${SYSDIR} && $svnversion 2>/dev/null` - case "$svn" in - [0-9]*) svn=" r${svn}" ;; - *) unset svn ;; - esac -fi - if [ -n "$git_cmd" ] ; then git=`$git_cmd rev-parse --verify --short HEAD 2>/dev/null` svn=`$git_cmd svn find-rev $git 2>/dev/null` -- 2.6.3 From 6f1ca2170452d12b9b0b48db6067530bf52f8eb0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 24 Sep 2015 14:34:45 +0200 Subject: [PATCH 27/34] sys/conf/newvers.sh: Do not add git hash from .git if KERNEL_VERSION_NUMBER is defined ... as this indicates that we are (trying to) reproduce a build. Do not use REPRO_SEED as it's always set nowadays. XXX: There should be a cleaner way to do this. --- sys/conf/newvers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/conf/newvers.sh b/sys/conf/newvers.sh index 9880cd5..e91efd4 100755 --- a/sys/conf/newvers.sh +++ b/sys/conf/newvers.sh @@ -114,7 +114,7 @@ else fi i=`${MAKE:-make} -V KERN_IDENT` -if [ -d "${SYSDIR}/../.git" ] ; then +if [ -z "${KERNEL_VERSION_NUMBER}" -a -d "${SYSDIR}/../.git" ] ; then for dir in /usr/bin /usr/local/bin; do if [ -x "${dir}/git" ] ; then git_cmd="${dir}/git --git-dir=${SYSDIR}/../.git" -- 2.6.3 From 46227eb93cf1132b71704d93c75106d9f1a6b446 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 17 Sep 2015 11:53:45 +0200 Subject: [PATCH 28/34] make-memstick.sh: Additionally fake uid and gid on the created fs --- release/amd64/make-memstick.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 534ec45..942d35b 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -29,7 +29,12 @@ if [ -e ${2} ]; then fi echo '/dev/ufs/FreeBSD_Install / ufs ro,noatime 1 1' > ${1}/etc/fstab -mtree -c -k time -p "${1}" | sed "s@time=.*@time=${EPOCH_DATE-0}.000000000@" > "${2}.mtree" +# Prepare mtree spec to fake timestamp, owner and group. +# As a result, man pages will be owned by root instead of man. +# Unfortunately we can't simply reuse ${1}/METALOG as it is incomplete. +mtree -c -k time -p "${1}" | sed \ + -e "s@time=.*@time=${EPOCH_DATE-0}.000000000 uname=root gname=wheel@" \ + > "${2}.mtree" makefs -B little -o label=FreeBSD_Install -F "${2}.mtree" ${2}.part ${1} if [ $? -ne 0 ]; then echo "makefs failed" -- 2.6.3 From 32a6ccffc97b1ac2d67bbc3b3102e7abb81040af Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 30 Sep 2015 13:15:44 +0200 Subject: [PATCH 29/34] release/amd64/make-memstick.sh: Error out if mkimg fails instead of cleaning up Makes debugging more convenient. --- release/amd64/make-memstick.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 942d35b..10ef336 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -44,5 +44,9 @@ rm ${1}/etc/fstab rm "${2}.mtree" mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} +if [ $? -ne 0 ]; then + echo "mkimg failed" + exit 1 +fi rm ${2}.part -- 2.6.3 From 5470f0e5b80e5a8142d6eb5d1c83b5bd2baeb2e3 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 4 Dec 2015 11:43:06 +0100 Subject: [PATCH 30/34] release/amd64/make-memstick.sh: Stop creating a puny swap partion --- release/amd64/make-memstick.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index 10ef336..f429068 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -43,7 +43,7 @@ fi rm ${1}/etc/fstab rm "${2}.mtree" -mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -p freebsd-swap::1M -o ${2} +mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} if [ $? -ne 0 ]; then echo "mkimg failed" exit 1 -- 2.6.3 From 5a52c4a10454faa4241da74732ecd5c556017e24 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Fri, 4 Dec 2015 11:44:10 +0100 Subject: [PATCH 31/34] release/amd64/make-memstick.sh: Stop creating an EFI partition as the EFI goo does not build reproducible --- release/amd64/make-memstick.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/amd64/make-memstick.sh b/release/amd64/make-memstick.sh index f429068..20c6bb1 100755 --- a/release/amd64/make-memstick.sh +++ b/release/amd64/make-memstick.sh @@ -43,7 +43,7 @@ fi rm ${1}/etc/fstab rm "${2}.mtree" -mkimg -y -s gpt -b ${1}/boot/pmbr -p efi:=${1}/boot/boot1.efifat -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} +mkimg -y -s gpt -b ${1}/boot/pmbr -p freebsd-boot:=${1}/boot/gptboot -p freebsd-ufs:=${2}.part -o ${2} if [ $? -ne 0 ]; then echo "mkimg failed" exit 1 -- 2.6.3 From 89635fab51709fad5d140e3dac3941efc9fdcc61 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Thu, 25 Jun 2015 17:26:59 +0200 Subject: [PATCH 32/34] Add reproduce.sh which makes reproducing ElectroBSD more convenient Squashed commits worth mentioning: - Add -j option to overwrite the maximum number make jobs - Assert that the source directory is untainted (according to strip-freebsd.sh) ... and add -a flag to remove offending files - Allow to resume a build by using the -r flag. --- reproduce.sh | 196 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100755 reproduce.sh diff --git a/reproduce.sh b/reproduce.sh new file mode 100755 index 0000000..dac0e34 --- /dev/null +++ b/reproduce.sh @@ -0,0 +1,196 @@ +#!/bin/sh + +########################################################################## +# Copyright (c) 2015 Fabian Keil +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +########################################################################## + +# reproduce.sh +# +# Script to make reproducing an ElectroBSD build more convenient. +# Before using it, make sure BUILD and EPOCH contain values other +# than __BUILD__ and __EPOCH__, either by editing the script or +# by putting them in a configuration file that is speficied with +# the -f option. + +# These variables have to be set to the values used for the build +# that is supposed to be reproduced. ${SRC_DIR} must contain the +# matching sources! +BUILD=__BUILD__ +EPOCH=__EPOCH__ + +# This is just a suggestion, feel free to overwrite it with the -j option. +MAX_MAKE_JOBS="${MAX_MAKE_JOBS-4}" + +# Currently hardcoded. +SRC_DIR=/usr/src +OPTIONAL_CONFIG_FILE="${SRC_DIR}/reproduce.conf" + +# Make sure we respawn with the same script, even if it is located +# outside the SRC_DIR and called with a relative path. +REPRODUCE_SH="$(realpath "${0}")" + +# When set to true, existing object files will be reused. +# If the source files changed, the result will not be reproducible! +RESUME_BUILD="${RESUME_BUILD-false}" + +announce_status() { + local msg \ + timestamp + + msg="${*}" + timestamp=$(date "+%Y-%m-%d_%H:%M") + + echo "${timestamp}: ${msg}" +} + +reproduce_all_the_things() { + + if "${RESUME_BUILD}"; then + announce_status "Resuming ..." + export KERNFAST=1 + export NO_CLEAN=1 + fi + + announce_status "Starting to build the kernel" + make buildkernel || return 1 + + announce_status "Starting to build the world" + make -j${MAX_MAKE_JOBS} buildworld || return 1 + + # Make sure obj files aren't dumped in ${SRC_DIR} + mkdir -p "/usr/obj${SRC_DIR}/release" || return 1 + + if ! "${RESUME_BUILD}"; then + announce_status "Starting to clean the release dir" + make -C "${SRC_DIR}/release" clean + fi + announce_status "Starting to build the release" + time make -C "${SRC_DIR}/release" memstick NO_FSCHG="yes" || return 1 + + announce_status "Done with release memstick for ${REPRO_SEED}" +} + +assert_untainted_source_tree() { + local auto_untaint \ + untaint_commands + + auto_untaint="${1}" + + untaint_commands="$(sh ./release/scripts/strip-freebsd.sh .)" + if [ -n "${untaint_commands}" ] ; then + if $auto_untaint; then + echo "Auto untainting $(pwd)" + echo "${untaint_commands}" | sh -x || return 1 + else + echo "${SRC_DIR} is tainted. Use -a flag to auto-untaint it." + return 1 + fi + fi + return 0 +} + +respawn_with_clean_environment() { + exec env -i PATH="/sbin:/bin:/usr/sbin:/usr/bin" HOME="/root" \ + LC_COLLATE=C SHELL=/bin/sh ALREADY_RESPAWNED=1 \ + MAX_MAKE_JOBS="${MAX_MAKE_JOBS}" RESUME_BUILD="${RESUME_BUILD}" \ + REPRO_SEED="${REPRO_SEED}" SRCCONF=/dev/null /bin/sh "${REPRODUCE_SH}" +} + +main() { + local args \ + auto_untaint config_file fake_user dry_run + + fake_user=elektropunker + + auto_untaint=false + dry_run=false + config_file="${OPTIONAL_CONFIG_FILE}" + + args=$(getopt af:j:nr $*) + if [ $? -ne 0 ]; then + echo 'You are doing it wrong: Invalid flag specified' + exit 2 + fi + set -- ${args} + while true; do + case "$1" in + -a) + shift + auto_untaint="true" + ;; + -j) + shift; + MAX_MAKE_JOBS="${1}" + shift; + ;; + -f) + shift; + config_file="${1}" + shift; + if [ ! -f "${config_file}" ]; then + echo "Config file ${config_file} does not exist" + exit 2 + fi + ;; + -n) + dry_run=true + shift + ;; + -r) + shift + RESUME_BUILD=true + ;; + --) + shift; break + ;; + esac + done + + if [ -f "${config_file}" ]; then + announce_status "Reading config from ${config_file}" + . "${config_file}" || exit 2 + fi + + if [ -n "${ALREADY_RESPAWNED}" -a "${ALREADY_RESPAWNED}" = 1 ]; then + if [ -z "${REPRO_SEED}" ]; then + announce_status "Respawned with REPRO_SEED unset" + return 1 + fi + reproduce_all_the_things + return + fi + if [ "${BUILD}" = "__BUILD__" ]; then + announce_status "BUILD not set" + return 1 + fi + if [ "${EPOCH}" = "__EPOCH__" ]; then + announce_status "EPOCH not set" + return 1 + fi + + export REPRO_SEED="${fake_user}:${BUILD}:${EPOCH}" + + announce_status "REPRO_SEED=${REPRO_SEED}" + + cd "${SRC_DIR}" || return 1 + + assert_untainted_source_tree "${auto_untaint}" || return 1 + + if ! $dry_run; then + respawn_with_clean_environment + fi +} + +main "${@}" -- 2.6.3 From ecad4f38eaac085d4ebda647f1e63f9aec728dc7 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 4 Nov 2015 18:55:18 +0100 Subject: [PATCH 33/34] share/doc: Detach 'legal' from the build It (tries to) install proprietary licenses for code that is not part of ElectroBSD and deleted by reproduce.sh's auto-untaint mode (-a). --- share/doc/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/share/doc/Makefile b/share/doc/Makefile index 7a02b29..5b5e507 100644 --- a/share/doc/Makefile +++ b/share/doc/Makefile @@ -5,7 +5,6 @@ SUBDIR= ${_IPv6} \ ${_atf} \ - legal \ ${_llvm} \ ${_pjdfstest} \ ${_roffdocs} -- 2.6.3 From c29e0c6307204a0000e6efa0e1613a03783952b0 Mon Sep 17 00:00:00 2001 From: Fabian Keil Date: Wed, 20 May 2015 13:38:21 +0200 Subject: [PATCH 34/34] sys/boot/Makefile.amd64: Detach efi from the built It doesn't built reproducible, among other things due to the embedded file system, and none of the ElectroBSD users I'm aware of have efi-capable systems anyway. --- sys/boot/Makefile.amd64 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/boot/Makefile.amd64 b/sys/boot/Makefile.amd64 index 384cf7a..586736d 100644 --- a/sys/boot/Makefile.amd64 +++ b/sys/boot/Makefile.amd64 @@ -1,6 +1,6 @@ # $FreeBSD$ -SUBDIR+= efi +#SUBDIR+= efi SUBDIR+= libstand32 SUBDIR+= zfs SUBDIR+= userboot -- 2.6.3