diff --git a/.github/workflows/bwrap.yml b/.github/workflows/bwrap.yml index 66cea846e..decdbcc49 100644 --- a/.github/workflows/bwrap.yml +++ b/.github/workflows/bwrap.yml @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -name: Run under bubblewrap +name: Run under (bubble)wrap on: workflow_dispatch: @@ -13,10 +13,15 @@ on: jobs: pass1: - name: Run up to Linux build under bubblewrap + name: Run up to Linux build under (bubble)wrap runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + virt-tool: [bwrap, wrap] steps: - name: Install bubblewrap + if: matrix.virt-tool == 'bwrap' run: sudo apt install bubblewrap - name: Checkout repo uses: actions/checkout@v3 @@ -43,7 +48,7 @@ jobs: distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1 + run: ./rootfs.py --${{matrix.virt-tool}} --external-sources --build-kernels --cores 2 --internal-ci pass1 - name: Archive created packages if: failure() # archive failed builds progress uses: actions/upload-artifact@v3 @@ -55,15 +60,20 @@ jobs: - name: Archive pass1_image uses: actions/upload-artifact@v3 with: - name: internal_pass1_image + name: internal_pass1_image_${{matrix.virt-tool}} path: pass1_image.tar pass2: - name: Run up to Python bootstrap under bubblewrap + name: Run up to Python bootstrap under (bubble)wrap needs: pass1 runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + virt-tool: [bwrap, wrap] steps: - name: Install bubblewrap + if: matrix.virt-tool == 'bwrap' run: sudo apt install bubblewrap - name: Checkout repo uses: actions/checkout@v3 @@ -75,7 +85,7 @@ jobs: - name: Get pass1_image uses: actions/download-artifact@v3 with: - name: internal_pass1_image + name: internal_pass1_image_${{matrix.virt-tool}} - name: Extract pass1_image run: tar -xf pass1_image.tar - name: Query cache for sources @@ -96,7 +106,7 @@ jobs: distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2 + run: ./rootfs.py --${{matrix.virt-tool}} --external-sources --build-kernels --cores 2 --internal-ci pass2 - name: Archive created packages if: failure() # archive failed builds progress uses: actions/upload-artifact@v3 @@ -108,15 +118,20 @@ jobs: - name: Archive pass2_image uses: actions/upload-artifact@v3 with: - name: internal_pass2_image + name: internal_pass2_image_${{matrix.virt-tool}} path: pass2_image.tar pass3: - name: Run remaining builds under bubblewrap + name: Run remaining builds under (bubble)wrap needs: pass2 runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + virt-tool: [bwrap, wrap] steps: - name: Install bubblewrap + if: matrix.virt-tool == 'bwrap' run: sudo apt install bubblewrap - name: Checkout repo uses: actions/checkout@v3 @@ -128,7 +143,7 @@ jobs: - name: Get pass2_image uses: actions/download-artifact@v3 with: - name: internal_pass2_image + name: internal_pass2_image_${{matrix.virt-tool}} - name: Extract pass2_image run: tar -xf pass2_image.tar - name: Query cache for sources @@ -149,10 +164,10 @@ jobs: distfiles key: cache-${{ hashFiles('steps/*/sources') }} - name: Run bootstrap - run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3 + run: ./rootfs.py --${{matrix.virt-tool}} --external-sources --build-kernels --cores 2 --internal-ci pass3 - name: Archive created packages if: always() # archive both failed and successful builds uses: actions/upload-artifact@v3 with: - name: packages + name: packages_${{matrix.virt-tool}} path: target/external/repo/** diff --git a/.reuse/dep5 b/.reuse/dep5 index 0b3e3bd6b..2f6bf5637 100644 --- a/.reuse/dep5 +++ b/.reuse/dep5 @@ -9,6 +9,6 @@ Source: https://github.com/fosslinux/live-bootstrap # Copyright: $YEAR $NAME <$CONTACT> # License: ... -Files: steps/*/sources steps/*/*.checksums steps/SHA256SUMS.pkgs steps/*/simple-patches/* seed/*.checksums +Files: steps/*/sources steps/*/*.checksums steps/SHA256SUMS.pkgs steps/*/simple-patches/* seed/*.checksums seed/wrap-bootstrap.cfg Copyright: none License: MIT diff --git a/lib/generator.py b/lib/generator.py index 2f69058b3..1e41f9d37 100755 --- a/lib/generator.py +++ b/lib/generator.py @@ -42,7 +42,13 @@ def reuse(self, target): self.external_dir = os.path.join(self.target_dir, 'external') self.distfiles() - def prepare(self, target, using_kernel=False, kernel_bootstrap=False, target_size=0): + # pylint: disable=too-many-arguments + def prepare(self, + target, + using_kernel=False, + kernel_bootstrap=False, + wrap=False, + target_size=0): """ Prepare basic media of live-bootstrap. /steps -- contains steps to be built @@ -82,7 +88,7 @@ def prepare(self, target, using_kernel=False, kernel_bootstrap=False, target_siz os.path.join(self.target_dir, 'kaem.x86')) else: self.stage0_posix(kernel_bootstrap) - self.seed() + self.seed(wrap) os.makedirs(self.external_dir) @@ -134,12 +140,17 @@ def stage0_posix(self, kernel_bootstrap=False): 'kaem-optional-seed') shutil.copy2(kaem_optional_seed, os.path.join(self.target_dir, 'init')) - def seed(self): + def seed(self, wrap): """Copy in extra seed files""" seed_dir = os.path.join(self.git_dir, 'seed') for entry in os.listdir(seed_dir): if os.path.isfile(os.path.join(seed_dir, entry)): shutil.copy2(os.path.join(seed_dir, entry), os.path.join(self.target_dir, entry)) + if wrap: + shutil.copy2(os.path.join(seed_dir, 'after-wrap.kaem'), + os.path.join(self.target_dir, 'after.kaem')) + shutil.copy2(os.path.join(seed_dir, 'after.kaem'), + os.path.join(self.target_dir, 'after-wrapped.kaem')) def distfiles(self): """Copy in distfiles""" diff --git a/rootfs.py b/rootfs.py index 1cfeab94a..87fb29d23 100755 --- a/rootfs.py +++ b/rootfs.py @@ -31,7 +31,7 @@ def create_configuration_file(args): config.write(f"ARCH={args.arch}\n") config.write(f"ARCH_DIR={stage0_arch_map.get(args.arch, args.arch)}\n") config.write(f"FORCE_TIMESTAMPS={args.force_timestamps}\n") - config.write(f"CHROOT={args.chroot or args.bwrap}\n") + config.write(f"CHROOT={args.chroot or args.bwrap or args.wrap}\n") config.write(f"UPDATE_CHECKSUMS={args.update_checksums}\n") config.write(f"JOBS={args.cores}\n") config.write(f"SWAP_SIZE={args.swap}\n") @@ -63,6 +63,8 @@ def main(): default="x86") parser.add_argument("-c", "--chroot", help="Run inside chroot", action="store_true") + parser.add_argument("-w", "--wrap", help="Run inside a minimal sandbox", + action="store_true") parser.add_argument("-bw", "--bwrap", help="Run inside a bwrap sandbox", action="store_true") parser.add_argument("-t", "--target", help="Target directory", @@ -127,15 +129,18 @@ def check_types(): count += 1 if args.bwrap: count += 1 + if args.wrap: + count += 1 if args.bare_metal: count += 1 return count if check_types() > 1: - raise ValueError("No more than one of qemu, chroot, bwrap, bare metal" - "may be used.") + raise ValueError("No more than one of qemu, chroot, bwrap, wrap, bare " + "metal may be used.") if check_types() == 0: - raise ValueError("One of qemu, chroot, bwrap, or bare metal must be selected.") + raise ValueError("One of qemu, chroot, bwrap, wrap, or bare metal must" + " be selected.") # Arch validation if args.arch != "x86": @@ -237,6 +242,17 @@ def bootstrap(args, generator, target, size): '--tmpfs', '/tmp', init) + elif args.wrap: + arch = stage0_arch_map.get(args.arch, args.arch) + if not args.internal_ci or args.internal_ci == "pass1": + generator.prepare(target, wrap = True) + arg_list = [os.path.join('bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')] + else: + generator.reuse(target) + arg_list = [os.path.join(arch, 'bin', 'wrap'), '/init'] + + run(*arg_list, cwd = generator.target_dir) + elif args.bare_metal: if args.kernel: generator.prepare(target, using_kernel=True, target_size=size) diff --git a/seed/after-wrap.kaem b/seed/after-wrap.kaem new file mode 100644 index 000000000..ebbfef4b2 --- /dev/null +++ b/seed/after-wrap.kaem @@ -0,0 +1,19 @@ +#!/bin/sh + +# SPDX-FileCopyrightText: 2024 Max Hearnden maxoscarhearnden@gmail.com +# +# SPDX-License-Identifier: GPL-3.0-or-later + +set -ex + +# detect wether we are in a rootfs.py environment +if ./${ARCH_DIR}/bin/catm steps/env-saved steps/env; then + ./${ARCH_DIR}/bin/wrap /${ARCH_DIR}/bin/kaem --file after-wrapped.kaem +else + # leave seed/stage0-posix + cd ../.. + + ARCH_DIR=seed/stage0-posix/${ARCH_DIR} + + ./${ARCH_DIR}/bin/wrap /${ARCH_DIR}/bin/kaem --file seed/after.kaem +fi diff --git a/seed/after.kaem b/seed/after.kaem index a9ab9aa05..b9dae1e32 100755 --- a/seed/after.kaem +++ b/seed/after.kaem @@ -11,5 +11,18 @@ set -ex PATH=/${ARCH_DIR}/bin -catm seed-full.kaem /steps/bootstrap.cfg /steps/env seed.kaem +if catm seed-full.kaem /steps/bootstrap.cfg /steps/env seed.kaem; then +else + replace --file /steps/env --output /steps/env --match-on /external/distfiles --replace-with /distfiles + cp /seed/wrap-bootstrap.cfg /steps/bootstrap.cfg + catm seed-full.kaem /steps/bootstrap.cfg /steps/env /seed/seed.kaem + cp /seed/configurator.c configurator.c + cp /seed/configurator.${ARCH}.checksums configurator.${ARCH}.checksums + + cp /seed/script-generator.c script-generator.c + cp /seed/script-generator.${ARCH}.checksums script-generator.${ARCH}.checksums + # placeholder value + FINAL_JOBS=1 +fi + kaem --file seed-full.kaem diff --git a/seed/seed.kaem b/seed/seed.kaem index 1468cc1d8..d56c480c4 100755 --- a/seed/seed.kaem +++ b/seed/seed.kaem @@ -64,7 +64,12 @@ MES_PKG=mes-0.27 MES_PREFIX=${SRCDIR}/${MES_PKG}/build/${MES_PKG} GUILE_LOAD_PATH=${MES_PREFIX}/mes/module:${MES_PREFIX}/module:${SRCDIR}/${MES_PKG}/build/${NYACC_PKG}/module -M2-Mesoplanet --architecture ${ARCH} -f configurator.c -o configurator +if M2-Mesoplanet --architecture ${ARCH} -f configurator.c -o configurator; then +else + # using lightweight wrapper + M2LIBC_PATH=/seed/stage0-posix/M2libc + M2-Mesoplanet --architecture ${ARCH} -f configurator.c -o configurator +fi # Checksums if match x${UPDATE_CHECKSUMS} xTrue; then sha256sum -o configurator.${ARCH}.checksums configurator diff --git a/seed/wrap-bootstrap.cfg b/seed/wrap-bootstrap.cfg new file mode 100644 index 000000000..6a827cada --- /dev/null +++ b/seed/wrap-bootstrap.cfg @@ -0,0 +1,5 @@ +CHROOT=True +DISK=sda1 +KERNEL_BOOTSTRAP=False +BUILD_KERNELS=False +JOBS=${FINAL_JOBS} diff --git a/steps/improve/merged_usr.sh b/steps/improve/merged_usr.sh index 14e79d87f..33d30748e 100755 --- a/steps/improve/merged_usr.sh +++ b/steps/improve/merged_usr.sh @@ -6,5 +6,9 @@ # Add the rest of the FHS that we will use and is not created pre-boot ln -s bin /usr/sbin for d in bin lib sbin; do + if [ -d "/${d}" ] && ! [ -L "/${d}" ]; then + # Move the non symlink directory out of the way + mv "/${d}" "/${d}-saved" + fi ln -s "usr/${d}" "/${d}" || true # these might exist if rerunning done diff --git a/steps/improve/setup_repo.sh b/steps/improve/setup_repo.sh index 60fc35128..cabf07a63 100755 --- a/steps/improve/setup_repo.sh +++ b/steps/improve/setup_repo.sh @@ -6,4 +6,4 @@ # mkdir -p /external/repo -tar -cf - --exclude='/external/repo/*' --exclude='/external/repo-preseeded/*' --exclude='/external/distfiles/*' --exclude='/dev/*' --exclude='/proc/*' --exclude='/sys/*' --exclude='/tmp/*' / | bzip2 --best > /external/repo/base.tar.bz2 +tar -cf - --exclude='/external/repo/*' --exclude='/external/repo-preseeded/*' --exclude='/external/distfiles/*' --exclude='/distfiles/*' --exclude='/dev/*' --exclude='/proc/*' --exclude='/sys/*' --exclude='/tmp/*' / | bzip2 --best > /external/repo/base.tar.bz2