bootstrap: add stage0 image reuse for guix and simplify guix handoff flow

- add --stage0-image in rootfs.py for qemu to boot an existing kernel-bootstrap image
- when --stage0-image is combined with --build-guix-also, update BUILD_GUIX_ALSO in image config and sync /steps-guix into the image
- require stage0 /init to contain guix handoff marker instead of patching /init implicitly
- add run_steps_guix_if_requested() to make_bootable-generated /init so rebooted stage0 images can enter steps-guix directly
- run /steps-guix/0.sh with bash in after.sh
- make script-generator start mode convention-based: /steps stays kaem-first, alternate roots (e.g. /steps-guix) start in bash
- remove redundant explicit /steps config-root argument from seed/preseeded/reconfigure script-generator calls

rootfs.py

@@ -17,6 +17,8 @@ you can run bootstap inside chroot.
 import argparse
 import os
 import signal
+import shutil
+import tempfile
 import threading
 
 from lib.generator import Generator, stage0_arch_map
@@ -24,6 +26,75 @@ from lib.simple_mirror import SimpleMirror
 from lib.target import Target
 from lib.utils import run, run_as_root
 
+def enable_stage0_guix(image_path):
+    """
+    Enable BUILD_GUIX_ALSO in an existing stage0 image and sync /steps-guix.
+    """
+    steps_guix_dir = os.path.abspath("steps-guix")
+    manifest = os.path.join(steps_guix_dir, "manifest")
+    if not os.path.isdir(steps_guix_dir) or not os.path.isfile(manifest):
+        raise ValueError("steps-guix/manifest does not exist while --build-guix-also is set.")
+
+    mountpoint = tempfile.mkdtemp(prefix="lb-stage0-", dir="/tmp")
+    mounted = False
+    try:
+        run_as_root(
+            "mount",
+            "-t", "ext4",
+            "-o", "loop,offset=1073741824",
+            image_path,
+            mountpoint,
+        )
+        mounted = True
+        script = '''
+import os
+import shutil
+import sys
+
+mountpoint = sys.argv[1]
+steps_guix_dir = sys.argv[2]
+
+config_path = os.path.join(mountpoint, "steps", "bootstrap.cfg")
+if not os.path.isfile(config_path):
+    raise SystemExit(f"Missing config in stage0 image: {config_path}")
+
+with open(config_path, "r", encoding="utf-8") as cfg:
+    lines = [line for line in cfg if not line.startswith("BUILD_GUIX_ALSO=")]
+lines.append("BUILD_GUIX_ALSO=True\\n")
+with open(config_path, "w", encoding="utf-8") as cfg:
+    cfg.writelines(lines)
+
+init_path = os.path.join(mountpoint, "init")
+if not os.path.isfile(init_path):
+    raise SystemExit(f"Missing /init in stage0 image: {init_path}")
+with open(init_path, "r", encoding="utf-8") as init_file:
+    if "run_steps_guix_if_requested()" not in init_file.read():
+        raise SystemExit(
+            "Stage0 image /init does not include guix handoff. "
+            "Rebuild once with current steps/improve/make_bootable.sh."
+        )
+
+dest_steps_guix = os.path.join(mountpoint, "steps-guix")
+if os.path.exists(dest_steps_guix):
+    shutil.rmtree(dest_steps_guix)
+shutil.copytree(steps_guix_dir, dest_steps_guix)
+'''
+        run_as_root("python3", "-c", script, mountpoint, steps_guix_dir)
+    finally:
+        if mounted:
+            run_as_root("umount", mountpoint)
+        os.rmdir(mountpoint)
+
+def prepare_stage0_work_image(base_image, output_dir, build_guix_also):
+    """
+    Copy stage0 base image to a disposable work image, optionally enabling guix.
+    """
+    work_image = os.path.join(output_dir, "stage0-work.img")
+    shutil.copy2(base_image, work_image)
+    if build_guix_also:
+        enable_stage0_guix(work_image)
+    return work_image
+
 def create_configuration_file(args):
     """
     Creates bootstrap.cfg file which would contain options used to
@@ -129,6 +200,8 @@ def main():
     parser.add_argument("-qs", "--target-size", help="Size of the target image (for QEMU only)",
                         default="16G")
     parser.add_argument("-qk", "--kernel", help="Custom early kernel to use")
+    parser.add_argument("--stage0-image",
+                        help="Boot an existing stage0 image (target/init.img) directly in QEMU")
 
     parser.add_argument("-b", "--bare-metal", help="Build images for bare metal",
                         action="store_true")
@@ -183,9 +256,18 @@ def main():
         args.swap = 0
 
     # Validate mirrors
-    if not args.mirrors:
+    if not args.mirrors and not args.stage0_image:
         raise ValueError("At least one mirror must be provided.")
 
+    if args.stage0_image:
+        if not args.qemu:
+            raise ValueError("--stage0-image can only be used with --qemu.")
+        if args.kernel:
+            raise ValueError("--stage0-image cannot be combined with --kernel.")
+        if not os.path.isfile(args.stage0_image):
+            raise ValueError(f"Stage0 image does not exist: {args.stage0_image}")
+        args.stage0_image = os.path.abspath(args.stage0_image)
+
     # Set constant umask
     os.umask(0o022)
 
@@ -203,15 +285,16 @@ def main():
                 mirror_servers.append(server)
 
     # bootstrap.cfg
-    try:
+    if not args.stage0_image:
-        os.remove(os.path.join('steps', 'bootstrap.cfg'))
+        try:
-    except FileNotFoundError:
+            os.remove(os.path.join('steps', 'bootstrap.cfg'))
-        pass
+        except FileNotFoundError:
-    if not args.no_create_config:
-        create_configuration_file(args)
-    else:
-        with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
             pass
+        if not args.no_create_config:
+            create_configuration_file(args)
+        else:
+            with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'):
+                pass
 
     # target
     target = Target(path=args.target)
@@ -227,12 +310,14 @@ def main():
             server.shutdown()
     signal.signal(signal.SIGINT, cleanup)
 
-    generator = Generator(arch=args.arch,
+    generator = None
-                          external_sources=args.external_sources,
+    if not args.stage0_image:
-                          repo_path=args.repo,
+        generator = Generator(arch=args.arch,
-                          early_preseed=args.early_preseed,
+                              external_sources=args.external_sources,
-                          mirrors=args.mirrors,
+                              repo_path=args.repo,
-                          build_guix_also=args.build_guix_also)
+                              early_preseed=args.early_preseed,
+                              mirrors=args.mirrors,
+                              build_guix_also=args.build_guix_also)
 
     bootstrap(args, generator, target, args.target_size, cleanup)
     cleanup()
@@ -299,7 +384,17 @@ print(shutil.which('chroot'))
             print(f"  1. Take {path}.img and write it to a boot drive and then boot it.")
 
     else:
-        if args.kernel:
+        if args.stage0_image:
+            work_image = prepare_stage0_work_image(args.stage0_image, target.path, args.build_guix_also)
+            arg_list = [
+                '-enable-kvm',
+                '-m', str(args.qemu_ram) + 'M',
+                '-smp', str(args.cores),
+                '-drive', 'file=' + work_image + ',format=raw',
+                '-machine', 'kernel-irqchip=split',
+                '-nic', 'user,ipv6=off,model=e1000'
+            ]
+        elif args.kernel:
             generator.prepare(target, using_kernel=True, target_size=size)
 
             arg_list = [

seed/preseeded.kaem

@@ -4,5 +4,5 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-/script-generator /steps/manifest /steps
+/script-generator /steps/manifest
 /usr/bin/kaem --file /preseed-jump.kaem

seed/script-generator.c

@@ -568,8 +568,11 @@ void generate(Directive *directives) {
 
 	int counter = 0;
 
-	/* Initially, we use kaem, not bash. */
+	/*
-	int bash_build = 0;
+	 * Default /steps manifests start in kaem.
+	 * Alternative step roots (for example /steps-guix) start in bash.
+	 */
+	int bash_build = (strcmp(steps_root, "/steps") == 0) ? 0 : 2;
 
 	FILE *out = start_script(counter, bash_build);
 	counter += 1;

seed/seed.kaem

@@ -80,6 +80,6 @@ if match x${UPDATE_CHECKSUMS} xTrue; then
 else
     sha256sum -c script-generator.${ARCH}.checksums
 fi
-./script-generator /steps/manifest /steps
+./script-generator /steps/manifest
 
 kaem --file /steps/0.sh

steps-guix/binutils-2.41/pass1.sh

@@ -0,0 +1,179 @@
+# SPDX-FileCopyrightText: 2022 Dor Askayo <dor.askayo@gmail.com>
+# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
+# SPDX-FileCopyrightText: 2021 Paul Dersey <pdersey@gmail.com>
+# SPDX-FileCopyrightText: 2023,2025 Samuel Tyler <samuel@samuelt.me>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# This binutils build targets the 64-bit kernel toolchain.
+: "${KERNEL_TARGET:=x86_64-unknown-linux-musl}"
+: "${KERNEL_SYSROOT:=/kernel-toolchain}"
+
+src_prepare() {
+    default
+
+    # Remove unused generated files
+    rm etc/Makefile.in etc/configure
+
+    # Remove unused parts
+    rm -r zlib
+
+    # intl/ Makefile is a bit broken because of new gettext
+    sed -i 's/@USE_INCLUDED_LIBINTL@/no/' intl/Makefile.in
+
+    # There is no way to add -all-static to libtool LDFLAGS (such a thing doesn't exist)
+    # -all-static is required for static binaries with libtool
+    sed -i 's:\(--mode=link $(CCLD)\):\1 -all-static:' {bfd,binutils,opcodes,ld,libctf,gas,gprof}/Makefile.in
+
+    # Rebuild bison files
+    touch -- */*.y
+    rm binutils/arparse.c binutils/arparse.h binutils/defparse.c \
+        binutils/defparse.h binutils/mcparse.c binutils/mcparse.h \
+        binutils/rcparse.c binutils/rcparse.h binutils/sysinfo.c \
+        binutils/sysinfo.h gas/config/bfin-parse.c gas/config/bfin-parse.h \
+        gas/config/loongarch-parse.c gas/config/loongarch-parse.h \
+        gas/config/m68k-parse.c gas/config/rl78-parse.c \
+        gas/config/rl78-parse.h gas/config/rx-parse.c gas/config/rx-parse.h \
+        gas/itbl-parse.c gas/itbl-parse.h gold/yyscript.c gold/yyscript.h \
+        intl/plural.c ld/deffilep.c ld/deffilep.h ld/ldgram.c ld/ldgram.h
+
+    # Rebuild flex generated files
+    touch -- */*.l */*/*.l
+    rm binutils/arlex.c binutils/deflex.c binutils/syslex.c \
+        gas/config/bfin-lex.c gas/config/loongarch-lex.c gas/itbl-lex.c \
+        ld/ldlex.c
+
+    # Remove prebuilt docs
+    find . -type f -name '*.info*' \
+                   -not -wholename './binutils/sysroff.info' \
+                   -delete
+    find . -type f \( -name '*.1' -or -name '*.man' \) -delete
+    rm libiberty/functions.texi
+
+    # Remove gettext translation files
+    find . -type f -name '*.gmo' -delete
+
+    # Remove pregenerated opcodes files
+    rm opcodes/i386-init.h opcodes/i386-tbl.h opcodes/i386-mnem.h \
+        opcodes/ia64-asmtab.c opcodes/z8k-opc.h opcodes/aarch64-asm-2.c \
+        opcodes/aarch64-opc-2.c opcodes/aarch64-dis-2.c \
+        opcodes/msp430-decode.c opcodes/rl78-decode.c opcodes/rx-decode.c
+    rm $(grep -l 'MACHINE GENERATED' opcodes/*.c opcodes/*.h)
+
+    # Various other blobs/generated headers
+    rm ld/emultempl/*.o_c
+    rm gprof/bsd_callg_bl.c gprof/flat_bl.c gprof/fsf_callg_bl.c
+    rm bfd/libcoff.h bfd/libbfd.h bfd/go32stub.h bfd/bfd-in2.h
+
+    # Generated testsuite stuff (xz-style attack)
+    rm libsframe/testsuite/libsframe.decode/DATA* \
+        ld/testsuite/ld-x86-64/*.obj.bz2 ld/testsuite/ld-sh/arch/*.s \
+        ld/testsuite/ld-sh/arch/arch_expected.txt \
+        ld/testsuite/ld-i386/pr27193a.o.bz2 \
+        gas/testsuite/gas/xstormy16/allinsn.sh \
+        gas/testsuite/gas/tic4x/opcodes.s gas/testsuite/gas/sh/arch/*.s \
+        gas/testsuite/gas/sh/arch/arch_expected.txt \
+        binutils/testsuite/binutils-all/x86-64/pr22451.o.bz2 \
+        binutils/testsuite/binutils-all/x86-64/pr26808.dwp.bz2 \
+        binutils/testsuite/binutils-all/x86-64/pr27708.exe.bz2 \
+        binutils/testsuite/binutils-all/nfp/*.nffw \
+        binutils/testsuite/binutils-all/pr26112.o.bz2 \
+        binutils/testsuite/binutils-all/pr26160.dwp.bz2
+
+    # Regenerate crc table in libiberty/crc32.c
+    cd libiberty
+    sed -n '/^   #include <stdio.h>/,/^   \}$/p' crc32.c > crcgen.c
+    gcc -o crcgen crcgen.c
+    sed '/crc_v3\.txt/{n; q}' crc32.c > crc32.c.new
+    ./crcgen >> crc32.c.new
+    sed '1,/^};$/d' crc32.c >> crc32.c.new
+    mv crc32.c.new crc32.c
+    cd ..
+
+    # bfd-in2.h is required to run autoreconf, but we don't have it yet
+    cd bfd
+    cp configure.ac configure.ac.bak
+    sed -i "s/bfd-in3.h:bfd-in2.h //" configure.ac
+    AUTOPOINT=true ACLOCAL=aclocal-1.15 AUTOMAKE=automake-1.15 autoreconf-2.69 -fi
+    ./configure
+    make headers
+    mv configure.ac.bak configure.ac
+    make distclean
+    cd ..
+
+    # Regenerate top-level (autogen + autotools)
+    autogen Makefile.def
+    ACLOCAL=aclocal-1.15 autoreconf-2.69 -fi
+
+    # Regenerate autoconf
+    for dir in bfd binutils gas gold gprof intl ld libctf libiberty libsframe opcodes; do
+        cd $dir
+        ACLOCAL=aclocal-1.15 AUTOMAKE=automake-1.15 autoreconf-2.69 -fi
+        cd ..
+    done
+    cd gprofng
+    # there is no libtool in gprofng, and libtoolize fails
+    LIBTOOLIZE=true ACLOCAL=aclocal-1.15 AUTOMAKE=automake-1.15 autoreconf-2.69 -fi
+    cd ..
+
+    # Rebuild dependencies in libiberty/Makefile.in
+    cd libiberty
+    ./configure --enable-maintainer-mode
+    make maint-deps
+    make distclean
+    cd ..
+
+    # Regenerate MeP sections
+    ./bfd/mep-relocs.pl
+}
+
+src_configure() {
+    mkdir build
+    cd build
+
+    LDFLAGS="-static" \
+    ../configure \
+        --prefix="${KERNEL_SYSROOT}" \
+        --libdir="${KERNEL_SYSROOT}/lib" \
+        --build="${TARGET}" \
+        --host="${TARGET}" \
+        --target="${KERNEL_TARGET}" \
+        --enable-static \
+        --disable-nls \
+        --disable-multilib \
+        --disable-plugins \
+        --disable-gprofng \
+        --enable-threads \
+        --enable-64-bit-bfd \
+        --enable-gold \
+        --enable-ld=default \
+        --enable-install-libiberty \
+        --enable-deterministic-archives \
+        --enable-targets=x86_64-elf,i386-elf \
+        --with-system-zlib \
+        --program-prefix="" \
+        --with-sysroot="${KERNEL_SYSROOT}" \
+        --srcdir=..
+}
+
+src_compile() {
+    make -C opcodes i386-gen
+    default_src_compile
+}
+
+src_install() {
+    make "${MAKEJOBS}" install \
+        DESTDIR="${DESTDIR}" \
+        prefix="${KERNEL_SYSROOT}" \
+        libdir="${KERNEL_SYSROOT}/lib"
+
+    # Create triplet symlinks
+    pushd "${DESTDIR}${KERNEL_SYSROOT}/bin"
+    for f in *; do
+        ln -sf "${KERNEL_SYSROOT}/bin/${f}" "${KERNEL_TARGET}-${f}"
+    done
+    popd
+
+    # FIXME: Binutils' manpages dates are not reproducible
+    rm -rf "${DESTDIR}${KERNEL_SYSROOT}/share/man"
+}

steps-guix/binutils-2.41/patches/new-gettext.patch

@@ -0,0 +1,17 @@
+SPDX-FileCopyrightText: 2023 Samuel Tyler <samuel@samuelt.me>
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+In new gettext external is required for AM_GNU_GETTEXT.
+
+--- binutils-2.41/intl/configure.ac	2023-02-07 18:57:56.350832016 +1100
++++ binutils-2.41/intl/configure.ac	2023-02-07 18:58:07.310054484 +1100
+@@ -4,7 +4,7 @@
+ AC_CONFIG_HEADER(config.h)
+ AC_CONFIG_MACRO_DIR(../config)
+ AM_GNU_GETTEXT_VERSION(0.12.1)
+-AM_GNU_GETTEXT([], [need-ngettext])
++AM_GNU_GETTEXT([external], [need-ngettext])
+ 
+ # This replaces the extensive use of DEFS in the original Makefile.in.
+ AC_DEFINE(IN_LIBINTL, 1, [Define because this is libintl.])

steps-guix/binutils-2.41/patches/no-maint-functions-texi.patch

@@ -0,0 +1,26 @@
+SPDX-FileCopyrightText: 2023 Samuel Tyler <samuel@samuelt.me>
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+Ensure functions.texi dependencies are satisfied.
+
+--- binutils-2.41/libiberty/Makefile.in	2023-12-08 15:18:57.985791235 +1100
++++ binutils-2.41/libiberty/Makefile.in	2023-12-08 15:19:15.391252344 +1100
+@@ -368,12 +368,12 @@
+ libiberty.html : $(srcdir)/libiberty.texi $(TEXISRC)
+ 	$(MAKEINFO) --no-split --html -I$(srcdir) -o $@ $<
+ 
+-@MAINT@$(srcdir)/functions.texi : stamp-functions
+-@MAINT@	@true
++$(srcdir)/functions.texi : stamp-functions
++	@true
+ 
+-@MAINT@stamp-functions : $(CFILES:%=$(srcdir)/%) $(TEXIFILES:%=$(srcdir)/%) $(srcdir)/gather-docs Makefile
+-@MAINT@@HAVE_PERL@	$(PERL) $(srcdir)/gather-docs $(srcdir) $(srcdir)/functions.texi $(CFILES) $(TEXIFILES)
+-@MAINT@	echo stamp > stamp-functions
++stamp-functions : $(CFILES:%=$(srcdir)/%) $(TEXIFILES:%=$(srcdir)/%) $(srcdir)/gather-docs Makefile
++@HAVE_PERL@	$(PERL) $(srcdir)/gather-docs $(srcdir) $(srcdir)/functions.texi $(CFILES) $(TEXIFILES)
++	echo stamp > stamp-functions
+ 
+ INSTALL_DEST = @INSTALL_DEST@
+ install: install_to_$(INSTALL_DEST) install-subdir

steps-guix/binutils-2.41/patches/no-maint-opcodes.patch

@@ -0,0 +1,18 @@
+SPDX-FileCopyrightText: 2022 Dor Askayo <dor.askayo@gmail.com>
+SPDX-FileCopyrightText: 2023 Samuel Tyler <samuel@samuelt.me>
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+Ensure i386-tbl.h dependencies are satisfied.
+
+--- binutils-2.41/opcodes/Makefile.am	2023-12-08 17:13:05.669136957 +1100
++++ binutils-2.41/opcodes/Makefile.am	2023-12-08 17:13:18.410480026 +1100
+@@ -540,7 +540,7 @@
+ # i386-gen will generate all headers in one go.  Use a pattern rule to properly
+ # express this, with the inner dash ('-') arbitrarily chosen to be the stem.
+ $(srcdir)/i386%tbl.h $(srcdir)/i386%init.h $(srcdir)/i386%mnem.h: \
+-		@MAINT@ i386-gen$(EXEEXT_FOR_BUILD) i386-opc.tbl i386-reg.tbl i386-opc.h
++		i386-gen$(EXEEXT_FOR_BUILD) i386-opc.tbl i386-reg.tbl i386-opc.h
+ 	$(AM_V_GEN)$(CPP) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) - \
+ 		< $(srcdir)/i386-opc.tbl \
+ 		| ./i386-gen$(EXEEXT_FOR_BUILD) --srcdir $(srcdir)

steps-guix/binutils-2.41/sources

@@ -0,0 +1 @@
+f https://mirrors.kernel.org/gnu/binutils/binutils-2.41.tar.xz ae9a5789e23459e59606e6714723f2d3ffc31c03174191ef0d015bdf06007450

steps-guix/helpers.sh

@@ -0,0 +1,617 @@
+#!/bin/bash -e
+
+# SPDX-FileCopyrightText: 2021 Andrius Štikonas <andrius@stikonas.eu>
+# SPDX-FileCopyrightText: 2021-22 Samuel Tyler <samuel@samuelt.me>
+# SPDX-FileCopyrightText: 2021 Paul Dersey <pdersey@gmail.com>
+# SPDX-FileCopyrightText: 2021 Melg Eight <public.melg8@gmail.com>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# Set constant umask
+umask 022
+
+# Get a list of files
+get_files() {
+    echo "."
+    _get_files "${1}"
+}
+
+_get_files() {
+    local prefix
+    prefix="${1}"
+    fs=
+    if [ -n "$(ls 2>/dev/null)" ]; then
+        fs=$(echo *)
+    fi
+    if [ -n "$(ls .[0-z]* 2>/dev/null)" ]; then
+        fs="${fs} $(echo .[0-z]*)"
+    fi
+    for f in ${fs}; do
+        # Archive symlinks to directories as symlinks
+        echo "${prefix}/${f}"
+        if [ -d "./${f}" ] && ! [ -h "./${f}" ]; then
+            cd "./${f}"
+            _get_files "${prefix}/${f}"
+            cd ..
+        fi
+    done
+}
+
+# Reset all timestamps to unix time 0
+reset_timestamp() {
+    if command -v find >/dev/null 2>&1; then
+        # find does not error out on exec error
+        find . -print0 | xargs -0 touch -h -t 197001010000.00
+    else
+        # A rudimentary find implementation that does the trick
+        fs=
+        if [ -n "$(ls 2>/dev/null)" ]; then
+            fs=$(echo ./*)
+        fi
+        if [ -n "$(ls .[0-z]* 2>/dev/null)" ]; then
+            fs="${fs} $(echo .[0-z]*)"
+        fi
+        for f in ${fs}; do
+            touch -h -t 197001010000.00 "./${f}"
+            if [ -d "./${f}" ]; then
+                cd "./${f}"
+                reset_timestamp
+                cd ..
+            fi
+        done
+    fi
+}
+
+# Fake grep
+_grep() {
+    local text="${1}"
+    local fname="${2}"
+    if command -v grep >/dev/null 2>&1; then
+        grep "${text}" "${fname}"
+    else
+        # shellcheck disable=SC2162
+        while read line; do
+            case "${line}" in *"${text}"*)
+                echo "${line}" ;;
+            esac
+        done < "${fname}"
+    fi
+}
+
+# Useful for perl extensions
+get_perl_version() {
+    perl -v | sed -n -re 's/.*[ (]v([0-9\.]*)[ )].*/\1/p'
+}
+
+get_revision() {
+    local pkg=$1
+    local oldpwd="${PWD}"
+    cd "/external/repo"
+    # Get revision (n time this package has been built)
+    revision=$( (ls -1 "${pkg}"* 2>/dev/null || true) | wc -l | sed 's/ *//g')
+    cd "${oldpwd}"
+}
+
+# Installs binary packages from an earlier run
+# This is useful to speed up development cycle
+bin_preseed() {
+    if [ -d "/external/repo-preseeded" ]; then
+        get_revision "${pkg}"
+        cd "/external/repo-preseeded"
+        test -e "${pkg}_${revision}.tar.bz2" || return 1
+        if [ "${UPDATE_CHECKSUMS}" = "True" ] || src_checksum "${pkg}" $((revision)); then
+            echo "${pkg}: installing prebuilt package."
+            mv "${pkg}_${revision}.tar.bz2" /external/repo || return 1
+            cd "/external/repo"
+            rm -f /tmp/filelist.txt
+            src_apply "${pkg}" $((revision))
+            cd "${SRCDIR}"
+            return 0
+        fi
+    fi
+    return 1
+}
+
+# Removes either an existing package or file
+uninstall() {
+    local in_fs in_pkg symlinks
+    while [ $# -gt 0 ]; do
+        removing="$1"
+        case "${removing}" in
+            /*)
+                # Removing a file
+                echo "removing file: ${removing}."
+                rm -f "${removing}"
+                ;;
+            *)
+                echo "${removing}: uninstalling."
+                local oldpwd="${PWD}"
+                mkdir -p "/tmp/removing"
+                cd "/tmp/removing"
+                get_revision "${removing}"
+                local filename="/external/repo/${removing}_$((revision-1)).tar.bz2"
+                # Initial bzip2 built against meslibc has broken pipes
+                bzip2 -dc "${filename}" | tar -xf -
+                # reverse to have files before directories
+                if command -v find >/dev/null 2>&1; then
+                    find . | sort -r > ../filelist
+                else
+                    get_files . | tac > ../filelist
+                fi
+                # shellcheck disable=SC2162
+                while read file; do
+                    if [ -d "${file}" ]; then
+                        if [ -z "$(ls -A "/${file}")" ]; then
+                            rmdir "/${file}"
+                        fi
+                    elif [ -h "${file}" ]; then
+                        symlinks="${symlinks} ${file}"
+                    else
+                        # in some cases we might be uninstalling a file that has already been overwritten
+                        # in this case we don't want to remove it
+                        in_fs="$(sha256sum "${file}" 2>/dev/null | cut -d' ' -f1)"
+                        in_pkg="$(sha256sum "/${file}" 2>/dev/null | cut -d' ' -f1)"
+                        if [ "${in_fs}" = "${in_pkg}" ]; then
+                            rm -f "/${file}"
+                        fi
+                    fi
+                done < ../filelist
+                rm -f ../filelist
+                for link in ${symlinks}; do
+                    if [ ! -e "/${link}" ]; then
+                        rm -f "/${link}"
+                    fi
+                done
+                cd "${oldpwd}"
+                rm -rf "/tmp/removing"
+                ;;
+        esac
+        shift
+    done
+}
+
+# Common build steps
+# Build function provides a few common stages with default implementation
+# that can be overridden on per package basis in the build script.
+# build takes two arguments:
+# 1) name-version of the package
+# 2) optionally specify build script. Default is pass$((revision+1)).sh
+# 3) optionally specify directory to cd into
+build() {
+    pkg=$1
+    get_revision "${pkg}"
+    script_name=${2:-pass$((revision+1)).sh}
+    dirname=${3:-${pkg}}
+
+    # shellcheck disable=SC2015
+    bin_preseed && return || true # Normal build if preseed fails
+
+    cd "${SRCDIR}/${pkg}" || (echo "Cannot cd into ${pkg}!"; kill $$)
+    echo "${pkg}: beginning build using script ${script_name}"
+    base_dir="${PWD}"
+    if [ -e "${base_dir}/patches-$(basename "${script_name}" .sh)" ]; then
+        patch_dir="${base_dir}/patches-$(basename "${script_name}" .sh)"
+    else
+        patch_dir="${base_dir}/patches"
+    fi
+    mk_dir="${base_dir}/mk"
+    files_dir="${base_dir}/files"
+
+    rm -rf "build"
+    mkdir "build"
+    cd "build"
+
+    build_script="${base_dir}/${script_name}"
+    if test -e "${build_script}"; then
+        # shellcheck source=/dev/null
+        . "${build_script}"
+    fi
+
+    echo "${pkg}: getting sources."
+    build_stage=src_get
+    call $build_stage
+
+    echo "${pkg}: unpacking source."
+    build_stage=src_unpack
+    call $build_stage
+    unset EXTRA_DISTFILES
+
+    cd "${dirname}" || (echo "Cannot cd into build/${dirname}!"; kill $$)
+
+    echo "${pkg}: preparing source."
+    build_stage=src_prepare
+    call $build_stage
+
+    echo "${pkg}: configuring source."
+    build_stage=src_configure
+    call $build_stage
+
+    echo "${pkg}: compiling source."
+    build_stage=src_compile
+    call $build_stage
+
+    echo "${pkg}: install to fakeroot."
+    mkdir -p "${DESTDIR}"
+    build_stage=src_install
+    call $build_stage
+
+    echo "${pkg}: postprocess binaries."
+    build_stage=src_postprocess
+    call $build_stage
+
+    echo "${pkg}: creating package."
+    cd "${DESTDIR}"
+    src_pkg
+
+    src_checksum "${pkg}" "${revision}"
+
+    echo "${pkg}: cleaning up."
+    rm -rf "${SRCDIR}/${pkg}/build"
+    rm -rf "${DESTDIR}"
+
+    echo "${pkg}: installing package."
+    src_apply "${pkg}" "${revision}"
+
+    echo "${pkg}: build successful"
+
+    cd "${SRCDIR}"
+
+    unset -f src_get src_unpack src_prepare src_configure src_compile src_install src_postprocess
+    unset extract
+}
+
+# An inventive way to randomise with what we know we always have
+randomize() {
+    if command -v shuf >/dev/null 2>&1; then
+        # shellcheck disable=SC2086
+        shuf -e ${1} | tr '\n' ' '
+    else
+        mkdir -p /tmp/random
+        for item in ${1}; do
+            touch --date=@${RANDOM} /tmp/random/"${item//\//~*~*}"
+        done
+        # cannot rely on find existing
+        # shellcheck disable=SC2012
+        ls -1 -t /tmp/random | sed 's:~\*~\*:/:g' | tr '\n' ' '
+        rm -r /tmp/random
+    fi
+}
+
+download_source_line() {
+    upstream_url="${1}"
+    checksum="${2}"
+    fname="${3}"
+    if ! [ -e "${fname}" ]; then
+        for mirror in $(randomize "${MIRRORS}"); do
+            # In qemu SimpleMirror is not running on the guest os, use qemu IP
+            case "${QEMU}-${mirror}" in 'True-http://127.0.0.1'*)
+                mirror="http://10.0.2.2${mirror#'http://127.0.0.1'}"
+            esac
+            mirror_url="${mirror}/${fname}"
+            echo "${mirror_url}"
+            curl --fail --retry 3 --location "${mirror_url}" --output "${fname}" || true && break
+        done
+        if ! [ -e "${fname}" ] && [ "${upstream_url}" != "_" ]; then
+            curl --fail --retry 3 --location "${upstream_url}" --output "${fname}" || true
+        fi
+    fi
+}
+
+check_source_line() {
+    url="${1}"
+    checksum="${2}"
+    fname="${3}"
+    if ! [ -e "${fname}" ]; then
+        echo "${fname} does not exist!"
+        false
+    fi
+    echo "${checksum}  ${fname}" > "${fname}.sum"
+    sha256sum -c "${fname}.sum"
+    rm "${fname}.sum"
+}
+
+source_line_action() {
+    action="$1"
+    shift
+    type="$1"
+    shift
+    case $type in
+        "g" | "git")
+            shift
+            ;;
+    esac
+    url="${1}"
+    checksum="${2}"
+    fname="${3}"
+    # Default to basename of url if not given
+    fname="${fname:-$(basename "${url}")}"
+    $action "$url" "$checksum" "$fname"
+}
+
+# Default get function that downloads source tarballs.
+default_src_get() {
+    # shellcheck disable=SC2153
+    cd "${DISTFILES}"
+    # shellcheck disable=SC2162
+    while read line; do
+        # This is intentional - we want to split out ${line} into separate arguments.
+        # shellcheck disable=SC2086
+        source_line_action download_source_line ${line}
+    done < "${base_dir}/sources"
+    # shellcheck disable=SC2162
+    while read line; do
+        # This is intentional - we want to split out ${line} into separate arguments.
+        # shellcheck disable=SC2086
+        source_line_action check_source_line ${line}
+    done < "${base_dir}/sources"
+    cd -
+}
+
+# Intelligently extracts a file based upon its filetype.
+extract_file() {
+    f="${3:-$(basename "${1}")}"
+    # shellcheck disable=SC2154
+    case "${noextract}" in
+        *${f}*)
+            cp "${DISTFILES}/${f}" .
+            ;;
+        *)
+            case "${f}" in
+                *.tar* | *.tgz)
+                    # shellcheck disable=SC2153
+                    if test -e "${PREFIX}/libexec/rmt"; then
+                        # Again, we want to split out into words.
+                        # shellcheck disable=SC2086
+                        tar --no-same-owner -xf "${DISTFILES}/${f}" ${extract}
+                    else
+                        # shellcheck disable=SC2086
+                        case "${f}" in
+                        *.tar.gz) tar -xzf "${DISTFILES}/${f}" ${extract} ;;
+                        *.tar.bz2)
+                            # Initial bzip2 built against meslibc has broken pipes
+                            bzip2 -dc "${DISTFILES}/${f}" | tar -xf - ${extract} ;;
+                        *.tar.xz | *.tar.lzma)
+                            if test -e "${PREFIX}/bin/xz"; then
+                                tar -xf "${DISTFILES}/${f}" --use-compress-program=xz ${extract}
+                            else
+                                unxz --file "${DISTFILES}/${f}" | tar -xf - ${extract}
+                            fi
+                            ;;
+                        esac
+                    fi
+                    ;;
+                *)
+                    cp "${DISTFILES}/${f}" .
+                    ;;
+            esac
+            ;;
+    esac
+}
+
+# Default unpacking function that unpacks all sources.
+default_src_unpack() {
+    # Handle the first one differently
+    first_line=$(head -n 1 ../sources)
+    # Again, we want to split out into words.
+    # shellcheck disable=SC2086
+    source_line_action extract_file ${first_line}
+    # This assumes there is only one directory in the tarball
+    # Get the dirname "smartly"
+    if ! [ -e "${dirname}" ]; then
+        for i in *; do
+            if [ -d "${i}" ]; then
+                dirname="${i}"
+                break
+            fi
+        done
+    fi
+    if ! [ -e "${dirname}" ]; then
+        # there are no directories extracted
+        dirname=.
+    fi
+    # shellcheck disable=SC2162
+    tail -n +2 ../sources | while read line; do
+        # shellcheck disable=SC2086
+        source_line_action extract_file ${line}
+    done
+}
+
+# Default function to prepare source code.
+# It applies all patches from patch_dir (at the moment only -p0 patches are supported).
+# Patches are applied from the parent directory.
+# Then it copies our custom makefile and any other custom files from files directory.
+default_src_prepare() {
+    if test -d "${patch_dir}"; then
+        if ls "${patch_dir}"/*.patch >/dev/null 2>&1; then
+            for p in "${patch_dir}"/*.patch; do
+                echo "Applying patch: ${p}"
+                patch -d.. -Np0 < "${p}"
+            done
+        fi
+    fi
+
+    makefile="${mk_dir}/main.mk"
+    if test -e "${makefile}"; then
+        cp "${makefile}" Makefile
+    fi
+
+    if test -d "${files_dir}"; then
+        cp --no-preserve=mode "${files_dir}"/* "${PWD}/"
+    fi
+}
+
+# Default function for configuring source.
+default_src_configure() {
+    :
+}
+
+# Default function for compiling source. It simply runs make without any parameters.
+default_src_compile() {
+    make "${MAKEJOBS}" -f Makefile PREFIX="${PREFIX}"
+}
+
+# Default installing function. PREFIX should be set by run.sh script.
+# Note that upstream makefiles might ignore PREFIX and have to be configured in configure stage.
+default_src_install() {
+    make -f Makefile install PREFIX="${PREFIX}" DESTDIR="${DESTDIR}"
+}
+
+# Helper function for permissions
+_do_strip() {
+    # shellcheck disable=SC2124
+    local f="${@: -1}"
+    if ! [ -w "${f}" ]; then
+        local perms
+        perms="$(stat -c %a "${f}")"
+        chmod u+w "${f}"
+    fi
+    strip "$@"
+    if [ -n "${perms}" ]; then
+        chmod "${perms}" "${f}"
+    fi
+}
+
+# Default function for postprocessing binaries.
+default_src_postprocess() {
+    if (command -v find && command -v file && command -v strip) >/dev/null 2>&1; then
+        # Logic largely taken from void linux 06-strip-and-debug-pkgs.sh
+        # shellcheck disable=SC2162
+        find "${DESTDIR}" -type f | while read f; do
+            case "$(file -bi "${f}")" in
+                application/x-executable*) _do_strip "${f}" ;;
+                application/x-sharedlib*|application/x-pie-executable*)
+                    machine_set="$(file -b "${f}")"
+                    case "${machine_set}" in
+                        *no\ machine*) ;; # don't strip ELF container-only
+                        *) _do_strip --strip-unneeded "${f}" ;;
+                    esac
+                    ;;
+                application/x-archive*) _do_strip --strip-debug "${f}" ;;
+            esac
+        done
+    fi
+}
+
+src_pkg() {
+    touch -t 197001010000.00 .
+    reset_timestamp
+
+    local tar_basename="${pkg}_${revision}.tar"
+    local dest_tar="/external/repo/${tar_basename}"
+    local filelist=/tmp/filelist.txt
+
+    cd /external/repo
+    # If grep is unavailable, then tar --sort is unavailable.
+    # So this does not need a command -v grep.
+    if tar --help | grep ' \-\-sort' >/dev/null 2>&1; then
+        tar -C "${DESTDIR}" --sort=name --hard-dereference \
+            --numeric-owner --owner=0 --group=0 --mode=go=rX,u+rw -cf "${dest_tar}" .
+    else
+        local olddir
+        olddir=$PWD
+        cd "${DESTDIR}"
+        local null
+        if command -v find >/dev/null 2>&1 && command -v sort >/dev/null 2>&1; then
+            find . -print0 | LC_ALL=C sort -z > "${filelist}"
+            null="--null"
+        elif command -v sort >/dev/null 2>&1; then
+            get_files .  | LC_ALL=C sort > "${filelist}"
+        else
+            get_files . > ${filelist}
+        fi
+        tar --no-recursion ${null} --files-from "${filelist}" \
+                --numeric-owner --owner=0 --group=0 --mode=go=rX,u+rw -cf "${dest_tar}"
+        rm -f "$filelist"
+        cd "$olddir"
+    fi
+    touch -t 197001010000.00 "${tar_basename}"
+    bzip2 --best "${tar_basename}"
+}
+
+src_checksum() {
+    local pkg=$1 revision=$2
+    local rval=0
+    if ! [ "$UPDATE_CHECKSUMS" = True ] ; then
+        # We avoid using pipes as that is not supported by initial sha256sum from mescc-tools-extra
+        local checksum_file=/tmp/checksum
+        _grep "${pkg}_${revision}.tar.bz2" "${SRCDIR}/SHA256SUMS.pkgs" > "${checksum_file}" || true
+        # Check there is something in checksum_file
+        if ! [ -s "${checksum_file}" ]; then
+            echo "${pkg}: no checksum stored!"
+            false
+        fi
+        echo "${pkg}: checksumming created package."
+        sha256sum -c "${checksum_file}" || rval=$?
+        rm "${checksum_file}"
+    fi
+    return "${rval}"
+}
+
+src_apply() {
+    local pkg="${1}" revision="${2}"
+    local TAR_PREFIX BZIP2_PREFIX
+
+    # Make sure we have at least one copy of tar
+    if [[ "${pkg}" == tar-* ]]; then
+        mkdir -p /tmp
+        cp "${PREFIX}/bin/tar" "/tmp/tar"
+        TAR_PREFIX="/tmp/"
+    fi
+
+    # Bash does not like to be overwritten
+    if [[ "${pkg}" == bash-* ]]; then
+        rm "${PREFIX}/bin/bash"
+    fi
+
+    # Overwriting files is mega busted, so do it manually
+    # shellcheck disable=SC2162
+    if [ -e /tmp/filelist.txt ]; then
+        while IFS= read -d $'\0' file; do
+            rm -f "/${file}" >/dev/null 2>&1 || true
+        done < /tmp/filelist.txt
+    fi
+
+    # Bzip2 does not like to be overwritten
+    if [[ "${pkg}" == bzip2-* ]]; then
+        mkdir -p /tmp
+        mv "${PREFIX}/bin/bzip2" "/tmp/bzip2"
+        BZIP2_PREFIX="/tmp/"
+    fi
+    "${BZIP2_PREFIX}bzip2" -dc "/external/repo/${pkg}_${revision}.tar.bz2" | \
+        "${TAR_PREFIX}tar" -C / -xpf -
+    rm -f "/tmp/bzip2" "/tmp/tar"
+}
+
+# Check if bash function exists
+fn_exists() {
+    test "$(type -t "$1")" == 'function'
+}
+
+# Call package specific function or default implementation.
+call() {
+    if fn_exists "$1"; then
+        $1
+    else
+        default_"${1}"
+    fi
+}
+
+# Call default build stage function
+default() {
+    "default_${build_stage}"
+}
+
+# Reusable helpers for sysroot-oriented stages (e.g. kernel toolchain/kernel build).
+sysroot_stage_init() {
+    : "${KERNEL_SYSROOT:=/kernel-toolchain}"
+}
+
+sysroot_src_install_default() {
+    # Generic install helper for packages that should install into a dedicated sysroot.
+    # Package-specific post-install actions stay in each package script.
+    sysroot_stage_init
+    make "${MAKEJOBS}" install \
+        DESTDIR="${DESTDIR}" \
+        prefix="${KERNEL_SYSROOT}" \
+        libdir="${KERNEL_SYSROOT}/lib"
+}

steps-guix/manifest

@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Guix extension manifest. Runs after main /steps manifest when BUILD_GUIX_ALSO=True.
+
+# We need a 64-bit kernel to enable Guix to run 64-bit programs.
+build: binutils-2.41
+

steps/improve/after.sh

@@ -27,7 +27,7 @@ if [ "${BUILD_GUIX_ALSO}" = True ]; then
     echo 'BUILD_GUIX_ALSO=False' >> /steps/bootstrap.cfg
 
     /script-generator /steps-guix/manifest /steps
-    kaem --file /steps-guix/0.sh
+    bash /steps-guix/0.sh
 fi
 
 if [ "${INTERACTIVE}" = True ]; then

steps/improve/make_bootable.sh

@@ -87,7 +87,32 @@ fi
 if [ "${CHROOT}" = False ]; then
     dhcpcd --waitip=4
 fi
+EOF
 
+if [ "${BUILD_GUIX_ALSO}" = True ]; then
+cat >> /init <<- 'EOF'
+run_steps_guix_if_requested() {
+    if [ "${BUILD_GUIX_ALSO}" != True ]; then
+        return 1
+    fi
+    if [ ! -f /steps-guix/manifest ]; then
+        echo "BUILD_GUIX_ALSO is True but /steps-guix/manifest is missing." >&2
+        exit 1
+    fi
+
+    sed -i '/^BUILD_GUIX_ALSO=/d' /steps/bootstrap.cfg
+    echo 'BUILD_GUIX_ALSO=False' >> /steps/bootstrap.cfg
+
+    /script-generator /steps-guix/manifest /steps
+    bash /steps-guix/0.sh
+    return 0
+}
+
+run_steps_guix_if_requested || true
+EOF
+fi
+
+cat >> /init <<- 'EOF'
 if [ "${QEMU}" = True ] && [ "${BARE_METAL}" = False ]; then
     if [ -c /dev/ttyS0 ]; then
         env - PATH=${PREFIX}/bin PS1="\w # " bash -i </dev/ttyS0 >/dev/ttyS0 2>&1

steps/improve/reconfigure.sh

@@ -3,4 +3,4 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 set -ex
 /configurator /steps/configurator
-/script-generator /steps/manifest /steps
+/script-generator /steps/manifest