WIP: Make ISO build (almost) reproducible, given the same set of input packages

Makefile

@@ -27,13 +27,16 @@ ifdef QUBES_RELEASE
     ISO_VERSION := $(QUBES_RELEASE)
     PUNGI_OPTS += --isfinal
 else
-    ISO_VERSION := $(shell date +%Y%m%d)
+    ISO_VERSION ?= $(shell date +%Y%m%d)
 endif
 PUNGI_OPTS += --ver="$(ISO_VERSION)"
 
 INSTALLER_KICKSTART ?= $(PWD)/conf/qubes-kickstart.cfg
 LIVE_KICKSTART ?= $(PWD)/conf/liveusb.ks
 
+SOURCE_DATE_EPOCH ?= $(shell git show -s --pretty=format:%at)
+export SOURCE_DATE_EPOCH
+
 help:
 	@echo "make iso              <== \o/";\
 	    echo; \

Makefile.builder

@@ -1,4 +1,5 @@
 RPM_SPEC_FILES.dom0 := \
+    squashfs-tools/squashfs-tools.spec \
     pykickstart/pykickstart.spec \
     blivet/python-blivet.spec \
     lorax/lorax.spec \

anaconda/dracut/anaconda-lib.sh

@@ -101,13 +101,30 @@ anaconda_live_root_dir() {
     anaconda_mount_sysroot $img
 }
 
+anaconda_mount_root_squashfs() {
+    local img="$1"
+
+    ROOTFLAGS="$(getarg rootflags)"
+
+    modprobe squashfs || die "squashfs not supported"
+    modprobe overlay || die "overlayfs not supported"
+    mkdir -m 0755 -p /run/overlayfs
+    mkdir -m 0755 -p /run/rootfsbase
+    mkdir -m 0755 -p /run/ovlwork
+
+    mount -r "$img" /run/rootfsbase
+
+    printf 'mount -t overlay LiveOS_rootfs -o%s,%s %s\n' "$ROOTFLAGS" \
+        'lowerdir=/run/rootfsbase,upperdir=/run/overlayfs,workdir=/run/ovlwork' \
+        "$NEWROOT" > $hookdir/mount/01-$$-live.sh
+    # satisfy wait_for_dev /dev/root
+    ln -s /dev/null /dev/root
+}
+
 anaconda_mount_sysroot() {
     local img="$1"
     if [ -e "$img" ]; then
-        /sbin/dmsquash-live-root $img
-        # dracut & systemd only mount things with root=live: so we have to do this ourselves
-        # See https://bugzilla.redhat.com/show_bug.cgi?id=1232411
-        printf 'mount /dev/mapper/live-rw %s\n' "$NEWROOT" > $hookdir/mount/01-$$-anaconda.sh
+        anaconda_mount_root_squashfs "$img"
     fi
 }
 

anaconda/dracut/module-setup.sh

@@ -7,7 +7,7 @@ check() {
 }
 
 depends() {
-    echo img-lib dmsquash-live
+    echo img-lib
     case "$(uname -m)" in
         s390*) echo cms ;;
     esac

lorax-templates-qubes/templates/efi.tmpl

@@ -4,6 +4,8 @@ EFIARCH_LOWER=efiarch.lower()
 EFIBOOTDIR="EFI/BOOT"
 APPLE_EFI_ICON=inroot+"/usr/share/pixmaps/bootloader/fedora.icns"
 APPLE_EFI_DISKNAME=inroot+"/usr/share/pixmaps/bootloader/fedora-media.vol"
+import os, time
+SOURCE_DATE_EPOCH = os.environ.get('SOURCE_DATE_EPOCH', str(int(time.time())))
 %>
 
 mkdir ${EFIBOOTDIR}
@@ -43,7 +45,7 @@ install boot/efi/EFI/*/fonts/unicode.pf2 ${EFIBOOTDIR}/fonts/
         runcmd chroot ${inroot} dracut --conf /dev/null --confdir /var/empty \
           --nomdadmconf --nolvmconf --nofscks --no-early-microcode \
           --no-hostonly --xz --install '/.buildstamp' \
-          --add 'anaconda pollcdrom dmsquash-live' \
+          --add 'anaconda pollcdrom' \
           --omit="${extra_dracut_modules}" --omit-drivers="${scsi_modules}" --omit-drivers="${extra_modules}" \
           /boot/efi/EFI/qubes/initrd-small.img ${kver}
         runcmd chroot ${inroot} rm -f /proc/modules
@@ -62,6 +64,8 @@ install boot/efi/EFI/*/fonts/unicode.pf2 ${EFIBOOTDIR}/fonts/
     runcmd mount ${outroot}/${img} ${outroot}/${EFIBOOTDIR} -o loop
     mkdir "${EFIBOOTDIR}/System Volume Information"
     install "${configdir}/System Volume Information/*" "${EFIBOOTDIR}/System Volume Information/"
+    runcmd find ${outroot}/${EFIBOOTDIR} -newermt "@${SOURCE_DATE_EPOCH}" -exec \
+        touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" {} +
     runcmd umount ${outroot}/${img}
     # verify if the image is under 32MB (max size of boot image on iso9660)
     runcmd sh -x -c '[ $(stat -c %s ${outroot}/${img}) -le 33554432 ]'

lorax-templates-qubes/templates/runtime-cleanup.tmpl

@@ -357,3 +357,16 @@ removepkg cdparanoia-libs opus libtheora libvisual flac-libs gsm avahi-glib avah
 
 ## metacity requires libvorbis and libvorbisfile, but enc/dec are no longer needed
 removefrom libvorbis --allbut /usr/${libdir}/libvorbisfile.* /usr/${libdir}/libvorbis.*
+
+## make the image more reproducible
+
+## make machine-id empty but present to avoid systemd populating /etc with
+## preset settings
+runcmd truncate -s 0 ${root}/etc/machine-id
+## journalctl message catalog, non-deterministic
+remove /var/lib/systemd/catalog/database
+## non-reproducible ldconfig cache
+remove /var/cache/ldconfig/aux-cache
+remove /etc/pki/ca-trust/extracted/java/cacerts
+remove /etc/group-
+remove /etc/gshadow-

lorax-templates-qubes/templates/runtime-postinstall.tmpl

@@ -6,6 +6,8 @@
 PYTHONDIR = sorted(glob("usr/"+libdir+"/python?.?"))[0]
 stubs = ("list-harddrives", "raidstart", "raidstop")
 configdir = configdir + "/common"
+import os, time
+SOURCE_DATE_EPOCH = os.environ.get('SOURCE_DATE_EPOCH', str(int(time.time())))
 %>
 
 ## move_stubs()
@@ -134,6 +136,20 @@ runcmd mknod ${root}/dev/null c 1 3
 runcmd mknod ${root}/dev/urandom c 1 9
 
 ## Record the package versions used to create the image
-runcmd chroot ${root} /bin/rpm -qa --pipe "tee /root/lorax-packages.log"
+runcmd chroot ${root} /bin/rpm -qa --pipe "sort | tee /root/lorax-packages.log"
+
+## fix fonconfig cache containing timestamps
+runcmd chroot ${root} /usr/bin/find /usr/share/fonts /usr/share/X11/fonts -newermt "@${SOURCE_DATE_EPOCH}" -exec \
+    touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" {} +
+runcmd chroot ${root} /usr/bin/fc-cache -f
+
+## drop timestamp from gconf.xml
+runcmd sed -i -e 's/mtime="[0-9]*" //' ${root}/etc/gconf/gconf.xml.defaults/desktop/gnome/interface/%gconf.xml
+
+## sort groups
+runcmd chroot ${root} /bin/sh -c "LC_ALL=C sort /etc/group > /etc/group.new && mv /etc/group.new /etc/group"
+runcmd chroot ${root} /bin/sh -c "LC_ALL=C sort /etc/gshadow > /etc/gshadow.new && mv /etc/gshadow.new /etc/gschadow"
+chmod /etc/gshadow 0700
+
 
 ## TODO: we could run prelink here if we wanted?

lorax/0005-Drop-inner-rootfs.img-layer.patch

@@ -0,0 +1,48 @@
+From cfc809df64e2778e51d47dc8b4466cf98efe8a2b Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
+ <[email protected]>
+Date: Wed, 3 Oct 2018 20:00:19 +0200
+Subject: [PATCH] Drop inner rootfs.img layer
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+Organization: Invisible Things Lab
+Cc: Marek Marczykowski-Górecki <[email protected]>
+
+Make runtime directly into squashfs image. This reduces largely
+unreproducible ext4 layer, but requires dracut module modification to
+properly mount the image.
+
+Signed-off-by: Marek Marczykowski-Górecki <[email protected]>
+---
+ src/pylorax/treebuilder.py | 12 +++---------
+ 1 file changed, 3 insertions(+), 9 deletions(-)
+
+diff --git a/src/pylorax/treebuilder.py b/src/pylorax/treebuilder.py
+index 5d4f8b7..b804c2f 100644
+--- a/src/pylorax/treebuilder.py
++++ b/src/pylorax/treebuilder.py
+@@ -212,17 +212,11 @@ class RuntimeBuilder(object):
+             generate_module_info(moddir+kver, outfile=moddir+"module-info")
+
+     def create_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", compressargs=None, size=2):
+-        # make live rootfs image - must be named "LiveOS/rootfs.img" for dracut
+         compressargs = compressargs or []
+-        workdir = joinpaths(os.path.dirname(outfile), "runtime-workdir")
+-        os.makedirs(joinpaths(workdir, "LiveOS"))
++        os.makedirs(os.path.dirname(outfile))
+
+-        imgutils.mkrootfsimg(self.vars.root, joinpaths(workdir, "LiveOS/rootfs.img"),
+-                             "Anaconda", size=size)
+-
+-        # squash the live rootfs and clean up workdir
+-        imgutils.mksquashfs(workdir, outfile, compression, compressargs)
+-        remove(workdir)
++        # squash the rootfs
++        imgutils.mksquashfs(self.vars.root, outfile, compression, compressargs)
+
+     def finished(self):
+         """ Done using RuntimeBuilder
+-- 
+2.17.1
+

lorax/0006-Use-SOURCE_DATE_EPOCH-for-metadata-timestamps.patch

@@ -0,0 +1,102 @@
+From c90eb097d7006378155e06a3d1e8148d61da90c5 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
+ <[email protected]>
+Date: Thu, 4 Oct 2018 18:16:34 +0200
+Subject: [PATCH 2/4] Use SOURCE_DATE_EPOCH for metadata timestamps
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+Organization: Invisible Things Lab
+Cc: Marek Marczykowski-Górecki <[email protected]>
+
+This include .buildinfo, .treeinfo and .discinfo.
+
+Signed-off-by: Marek Marczykowski-Górecki <[email protected]>
+---
+ src/pylorax/buildstamp.py | 7 ++++++-
+ src/pylorax/discinfo.py   | 8 +++++++-
+ src/pylorax/treeinfo.py   | 8 +++++++-
+ 3 files changed, 20 insertions(+), 3 deletions(-)
+
+diff --git a/src/pylorax/buildstamp.py b/src/pylorax/buildstamp.py
+index 4219944..4376784 100644
+--- a/src/pylorax/buildstamp.py
++++ b/src/pylorax/buildstamp.py
+@@ -23,6 +23,7 @@ import logging
+ logger = logging.getLogger("pylorax.buildstamp")
+
+ import datetime
++import os
+
+
+ class BuildStamp(object):
+@@ -33,7 +34,11 @@ class BuildStamp(object):
+         self.bugurl = bugurl
+         self.isfinal = isfinal
+
+-        now = datetime.datetime.now()
++        if 'SOURCE_DATE_EPOCH' in os.environ:
++            now = datetime.datetime.utcfromtimestamp(
++                int(os.environ['SOURCE_DATE_EPOCH']))
++        else:
++            now = datetime.datetime.now()
+         now = now.strftime("%Y%m%d%H%M")
+         self.uuid = "{0}.{1}".format(now, buildarch)
+
+diff --git a/src/pylorax/discinfo.py b/src/pylorax/discinfo.py
+index 9dad83b..311bae3 100644
+--- a/src/pylorax/discinfo.py
++++ b/src/pylorax/discinfo.py
+@@ -22,6 +22,7 @@
+ import logging
+ logger = logging.getLogger("pylorax.discinfo")
+
++import os
+ import time
+
+
+@@ -32,8 +33,13 @@ class DiscInfo(object):
+         self.basearch = basearch
+
+     def write(self, outfile):
++        if 'SOURCE_DATE_EPOCH' in os.environ:
++            timestamp = int(os.environ['SOURCE_DATE_EPOCH'])
++        else:
++            timestamp = time.time()
++
+         logger.info("writing .discinfo file")
+         with open(outfile, "w") as fobj:
+-            fobj.write("{0:f}\n".format(time.time()))
++            fobj.write("{0:f}\n".format(timestamp))
+             fobj.write("{0.release}\n".format(self))
+             fobj.write("{0.basearch}\n".format(self))
+diff --git a/src/pylorax/treeinfo.py b/src/pylorax/treeinfo.py
+index 4c84006..cc1ad3f 100644
+--- a/src/pylorax/treeinfo.py
++++ b/src/pylorax/treeinfo.py
+@@ -23,6 +23,7 @@ import logging
+ logger = logging.getLogger("pylorax.treeinfo")
+
+ import configparser
++import os
+ import time
+
+
+@@ -33,8 +34,13 @@ class TreeInfo(object):
+
+         self.c = configparser.ConfigParser()
+
++        if 'SOURCE_DATE_EPOCH' in os.environ:
++            timestamp = os.environ['SOURCE_DATE_EPOCH']
++        else:
++            timestamp = str(time.time())
++
+         section = "general"
+-        data = {"timestamp": str(time.time()),
++        data = {"timestamp": timestamp,
+                 "family": product,
+                 "version": version,
+                 "name": "%s-%s" % (product, version),
+-- 
+2.17.1
+

lorax/0007-Preserve-timestamps-when-building-fs-image.patch

@@ -0,0 +1,34 @@
+From 3457b203feac0af5ee5c388a6c0351978dadcc1a Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
+ <[email protected]>
+Date: Fri, 5 Oct 2018 04:48:09 +0200
+Subject: [PATCH 3/4] Preserve timestamps when building fs image
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+Organization: Invisible Things Lab
+Cc: Marek Marczykowski-Górecki <[email protected]>
+
+Even when FS do not support owner/modes, preserve timestamps.
+
+Signed-off-by: Marek Marczykowski-Górecki <[email protected]>
+---
+ src/pylorax/imgutils.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/pylorax/imgutils.py b/src/pylorax/imgutils.py
+index 25c300d..942695f 100644
+--- a/src/pylorax/imgutils.py
++++ b/src/pylorax/imgutils.py
+@@ -219,7 +219,7 @@ def copytree(src, dest, preserve=True):
+     If preserve is False, uses cp -R (useful for modeless filesystems)
+     raises CalledProcessError if copy fails.'''
+     logger.debug("copytree %s %s", src, dest)
+-    cp = ["cp", "-a"] if preserve else ["cp", "-R", "-L"]
++    cp = ["cp", "-a"] if preserve else ["cp", "-R", "-L", "--preserve=timestamps"]
+     cp += [join(src, "."), os.path.abspath(dest)]
+     runcmd(cp)
+
+-- 
+2.17.1
+

lorax/0008-Use-SOURCE_DATE_EPOCH-for-volumeid-of-efi-boot-image.patch

@@ -0,0 +1,40 @@
+From 7e29418e9a5692c1f5ff7327929cd48f543d3d80 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
+ <[email protected]>
+Date: Fri, 5 Oct 2018 04:48:57 +0200
+Subject: [PATCH 4/4] Use SOURCE_DATE_EPOCH for volumeid of efi boot image
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+Organization: Invisible Things Lab
+Cc: Marek Marczykowski-Górecki <[email protected]>
+
+By default mkfs.mksdos choose volume id based on current time. If
+SOURCE_DATE_EPOCH is set, use that instead.
+
+Signed-off-by: Marek Marczykowski-Górecki <[email protected]>
+---
+ src/pylorax/imgutils.py | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/src/pylorax/imgutils.py b/src/pylorax/imgutils.py
+index 6cd67e0..92de296 100644
+--- a/src/pylorax/imgutils.py
++++ b/src/pylorax/imgutils.py
+@@ -398,8 +398,12 @@ def mkfsimage(fstype, rootdir, outfile, size=None, mkfsargs=None, mountargs="",
+ # convenience functions with useful defaults
+ def mkdosimg(rootdir, outfile, size=None, label="", mountargs="shortname=winnt,umask=0077", graft=None):
+     graft = graft or {}
++    mkfsargs = ["-n", label]
++    if 'SOURCE_DATE_EPOCH' in os.environ:
++        mkfsargs.extend(["-i",
++                "{:x}".format(int(os.environ['SOURCE_DATE_EPOCH']))])
+     mkfsimage("msdos", rootdir, outfile, size, mountargs=mountargs,
+-              mkfsargs=["-n", label], graft=graft)
++              mkfsargs=mkfsargs, graft=graft)
+
+ def mkext4img(rootdir, outfile, size=None, label="", mountargs="", graft=None):
+     graft = graft or {}
+-- 
+2.17.1
+

lorax/lorax.spec

@@ -20,6 +20,10 @@ Patch1:         0001-Allow-specify-gpg-key-for-a-repository.patch
 Patch2:         0002-verify-packages-signature.patch
 Patch3:         0003-Update-package-verification-for-dnf-API.patch
 Patch4:         0004-Remove-branding-code.patch
+Patch5:         0005-Drop-inner-rootfs.img-layer.patch
+Patch6:         0006-Use-SOURCE_DATE_EPOCH-for-metadata-timestamps.patch
+Patch7:         0007-Preserve-timestamps-when-building-fs-image.patch
+Patch8:         0008-Use-SOURCE_DATE_EPOCH-for-volumeid-of-efi-boot-image.patch
 
 BuildRequires:  python3-devel
 
@@ -126,6 +130,10 @@ Lorax templates for creating the boot.iso and live isos are placed in
 %patch2 -p1
 %patch3 -p1
 %patch4 -p1
+%patch5 -p1
+%patch6 -p1
+%patch7 -p1
+%patch8 -p1
 
 %build