diff --git a/rootfs.py b/rootfs.py index 6634fce4..7c3b756c 100755 --- a/rootfs.py +++ b/rootfs.py @@ -17,6 +17,8 @@ you can run bootstap inside chroot. import argparse import os import signal +import shutil +import tempfile import threading from lib.generator import Generator, stage0_arch_map @@ -24,6 +26,75 @@ from lib.simple_mirror import SimpleMirror from lib.target import Target from lib.utils import run, run_as_root +def enable_stage0_guix(image_path): + """ + Enable BUILD_GUIX_ALSO in an existing stage0 image and sync /steps-guix. + """ + steps_guix_dir = os.path.abspath("steps-guix") + manifest = os.path.join(steps_guix_dir, "manifest") + if not os.path.isdir(steps_guix_dir) or not os.path.isfile(manifest): + raise ValueError("steps-guix/manifest does not exist while --build-guix-also is set.") + + mountpoint = tempfile.mkdtemp(prefix="lb-stage0-", dir="/tmp") + mounted = False + try: + run_as_root( + "mount", + "-t", "ext4", + "-o", "loop,offset=1073741824", + image_path, + mountpoint, + ) + mounted = True + script = ''' +import os +import shutil +import sys + +mountpoint = sys.argv[1] +steps_guix_dir = sys.argv[2] + +config_path = os.path.join(mountpoint, "steps", "bootstrap.cfg") +if not os.path.isfile(config_path): + raise SystemExit(f"Missing config in stage0 image: {config_path}") + +with open(config_path, "r", encoding="utf-8") as cfg: + lines = [line for line in cfg if not line.startswith("BUILD_GUIX_ALSO=")] +lines.append("BUILD_GUIX_ALSO=True\\n") +with open(config_path, "w", encoding="utf-8") as cfg: + cfg.writelines(lines) + +init_path = os.path.join(mountpoint, "init") +if not os.path.isfile(init_path): + raise SystemExit(f"Missing /init in stage0 image: {init_path}") +with open(init_path, "r", encoding="utf-8") as init_file: + if "run_steps_guix_if_requested()" not in init_file.read(): + raise SystemExit( + "Stage0 image /init does not include guix handoff. " + "Rebuild once with current steps/improve/make_bootable.sh." + ) + +dest_steps_guix = os.path.join(mountpoint, "steps-guix") +if os.path.exists(dest_steps_guix): + shutil.rmtree(dest_steps_guix) +shutil.copytree(steps_guix_dir, dest_steps_guix) +''' + run_as_root("python3", "-c", script, mountpoint, steps_guix_dir) + finally: + if mounted: + run_as_root("umount", mountpoint) + os.rmdir(mountpoint) + +def prepare_stage0_work_image(base_image, output_dir, build_guix_also): + """ + Copy stage0 base image to a disposable work image, optionally enabling guix. + """ + work_image = os.path.join(output_dir, "stage0-work.img") + shutil.copy2(base_image, work_image) + if build_guix_also: + enable_stage0_guix(work_image) + return work_image + def create_configuration_file(args): """ Creates bootstrap.cfg file which would contain options used to @@ -129,6 +200,8 @@ def main(): parser.add_argument("-qs", "--target-size", help="Size of the target image (for QEMU only)", default="16G") parser.add_argument("-qk", "--kernel", help="Custom early kernel to use") + parser.add_argument("--stage0-image", + help="Boot an existing stage0 image (target/init.img) directly in QEMU") parser.add_argument("-b", "--bare-metal", help="Build images for bare metal", action="store_true") @@ -183,9 +256,18 @@ def main(): args.swap = 0 # Validate mirrors - if not args.mirrors: + if not args.mirrors and not args.stage0_image: raise ValueError("At least one mirror must be provided.") + if args.stage0_image: + if not args.qemu: + raise ValueError("--stage0-image can only be used with --qemu.") + if args.kernel: + raise ValueError("--stage0-image cannot be combined with --kernel.") + if not os.path.isfile(args.stage0_image): + raise ValueError(f"Stage0 image does not exist: {args.stage0_image}") + args.stage0_image = os.path.abspath(args.stage0_image) + # Set constant umask os.umask(0o022) @@ -203,15 +285,16 @@ def main(): mirror_servers.append(server) # bootstrap.cfg - try: - os.remove(os.path.join('steps', 'bootstrap.cfg')) - except FileNotFoundError: - pass - if not args.no_create_config: - create_configuration_file(args) - else: - with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'): + if not args.stage0_image: + try: + os.remove(os.path.join('steps', 'bootstrap.cfg')) + except FileNotFoundError: pass + if not args.no_create_config: + create_configuration_file(args) + else: + with open(os.path.join('steps', 'bootstrap.cfg'), 'a', encoding='UTF-8'): + pass # target target = Target(path=args.target) @@ -227,12 +310,14 @@ def main(): server.shutdown() signal.signal(signal.SIGINT, cleanup) - generator = Generator(arch=args.arch, - external_sources=args.external_sources, - repo_path=args.repo, - early_preseed=args.early_preseed, - mirrors=args.mirrors, - build_guix_also=args.build_guix_also) + generator = None + if not args.stage0_image: + generator = Generator(arch=args.arch, + external_sources=args.external_sources, + repo_path=args.repo, + early_preseed=args.early_preseed, + mirrors=args.mirrors, + build_guix_also=args.build_guix_also) bootstrap(args, generator, target, args.target_size, cleanup) cleanup() @@ -299,7 +384,17 @@ print(shutil.which('chroot')) print(f" 1. Take {path}.img and write it to a boot drive and then boot it.") else: - if args.kernel: + if args.stage0_image: + work_image = prepare_stage0_work_image(args.stage0_image, target.path, args.build_guix_also) + arg_list = [ + '-enable-kvm', + '-m', str(args.qemu_ram) + 'M', + '-smp', str(args.cores), + '-drive', 'file=' + work_image + ',format=raw', + '-machine', 'kernel-irqchip=split', + '-nic', 'user,ipv6=off,model=e1000' + ] + elif args.kernel: generator.prepare(target, using_kernel=True, target_size=size) arg_list = [ diff --git a/seed/preseeded.kaem b/seed/preseeded.kaem index 25510bc9..0d92eaa4 100755 --- a/seed/preseeded.kaem +++ b/seed/preseeded.kaem @@ -4,5 +4,5 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -/script-generator /steps/manifest /steps +/script-generator /steps/manifest /usr/bin/kaem --file /preseed-jump.kaem diff --git a/seed/script-generator.c b/seed/script-generator.c index 6c193178..62bce6ac 100644 --- a/seed/script-generator.c +++ b/seed/script-generator.c @@ -568,8 +568,11 @@ void generate(Directive *directives) { int counter = 0; - /* Initially, we use kaem, not bash. */ - int bash_build = 0; + /* + * Default /steps manifests start in kaem. + * Alternative step roots (for example /steps-guix) start in bash. + */ + int bash_build = (strcmp(steps_root, "/steps") == 0) ? 0 : 2; FILE *out = start_script(counter, bash_build); counter += 1; diff --git a/seed/seed.kaem b/seed/seed.kaem index 9f9af4d2..5fd74491 100755 --- a/seed/seed.kaem +++ b/seed/seed.kaem @@ -80,6 +80,6 @@ if match x${UPDATE_CHECKSUMS} xTrue; then else sha256sum -c script-generator.${ARCH}.checksums fi -./script-generator /steps/manifest /steps +./script-generator /steps/manifest kaem --file /steps/0.sh diff --git a/steps-guix/.DS_Store b/steps-guix/.DS_Store new file mode 100644 index 00000000..7fdb7200 Binary files /dev/null and b/steps-guix/.DS_Store differ diff --git a/steps-guix/binutils-2.41/pass1.sh b/steps-guix/binutils-2.41/pass1.sh new file mode 100755 index 00000000..b68fd9db --- /dev/null +++ b/steps-guix/binutils-2.41/pass1.sh @@ -0,0 +1,179 @@ +# SPDX-FileCopyrightText: 2022 Dor Askayo +# SPDX-FileCopyrightText: 2021 Andrius Štikonas +# SPDX-FileCopyrightText: 2021 Paul Dersey +# SPDX-FileCopyrightText: 2023,2025 Samuel Tyler +# +# SPDX-License-Identifier: GPL-3.0-or-later + +# This binutils build targets the 64-bit kernel toolchain. +: "${KERNEL_TARGET:=x86_64-unknown-linux-musl}" +: "${KERNEL_SYSROOT:=/kernel-toolchain}" + +src_prepare() { + default + + # Remove unused generated files + rm etc/Makefile.in etc/configure + + # Remove unused parts + rm -r zlib + + # intl/ Makefile is a bit broken because of new gettext + sed -i 's/@USE_INCLUDED_LIBINTL@/no/' intl/Makefile.in + + # There is no way to add -all-static to libtool LDFLAGS (such a thing doesn't exist) + # -all-static is required for static binaries with libtool + sed -i 's:\(--mode=link $(CCLD)\):\1 -all-static:' {bfd,binutils,opcodes,ld,libctf,gas,gprof}/Makefile.in + + # Rebuild bison files + touch -- */*.y + rm binutils/arparse.c binutils/arparse.h binutils/defparse.c \ + binutils/defparse.h binutils/mcparse.c binutils/mcparse.h \ + binutils/rcparse.c binutils/rcparse.h binutils/sysinfo.c \ + binutils/sysinfo.h gas/config/bfin-parse.c gas/config/bfin-parse.h \ + gas/config/loongarch-parse.c gas/config/loongarch-parse.h \ + gas/config/m68k-parse.c gas/config/rl78-parse.c \ + gas/config/rl78-parse.h gas/config/rx-parse.c gas/config/rx-parse.h \ + gas/itbl-parse.c gas/itbl-parse.h gold/yyscript.c gold/yyscript.h \ + intl/plural.c ld/deffilep.c ld/deffilep.h ld/ldgram.c ld/ldgram.h + + # Rebuild flex generated files + touch -- */*.l */*/*.l + rm binutils/arlex.c binutils/deflex.c binutils/syslex.c \ + gas/config/bfin-lex.c gas/config/loongarch-lex.c gas/itbl-lex.c \ + ld/ldlex.c + + # Remove prebuilt docs + find . -type f -name '*.info*' \ + -not -wholename './binutils/sysroff.info' \ + -delete + find . -type f \( -name '*.1' -or -name '*.man' \) -delete + rm libiberty/functions.texi + + # Remove gettext translation files + find . -type f -name '*.gmo' -delete + + # Remove pregenerated opcodes files + rm opcodes/i386-init.h opcodes/i386-tbl.h opcodes/i386-mnem.h \ + opcodes/ia64-asmtab.c opcodes/z8k-opc.h opcodes/aarch64-asm-2.c \ + opcodes/aarch64-opc-2.c opcodes/aarch64-dis-2.c \ + opcodes/msp430-decode.c opcodes/rl78-decode.c opcodes/rx-decode.c + rm $(grep -l 'MACHINE GENERATED' opcodes/*.c opcodes/*.h) + + # Various other blobs/generated headers + rm ld/emultempl/*.o_c + rm gprof/bsd_callg_bl.c gprof/flat_bl.c gprof/fsf_callg_bl.c + rm bfd/libcoff.h bfd/libbfd.h bfd/go32stub.h bfd/bfd-in2.h + + # Generated testsuite stuff (xz-style attack) + rm libsframe/testsuite/libsframe.decode/DATA* \ + ld/testsuite/ld-x86-64/*.obj.bz2 ld/testsuite/ld-sh/arch/*.s \ + ld/testsuite/ld-sh/arch/arch_expected.txt \ + ld/testsuite/ld-i386/pr27193a.o.bz2 \ + gas/testsuite/gas/xstormy16/allinsn.sh \ + gas/testsuite/gas/tic4x/opcodes.s gas/testsuite/gas/sh/arch/*.s \ + gas/testsuite/gas/sh/arch/arch_expected.txt \ + binutils/testsuite/binutils-all/x86-64/pr22451.o.bz2 \ + binutils/testsuite/binutils-all/x86-64/pr26808.dwp.bz2 \ + binutils/testsuite/binutils-all/x86-64/pr27708.exe.bz2 \ + binutils/testsuite/binutils-all/nfp/*.nffw \ + binutils/testsuite/binutils-all/pr26112.o.bz2 \ + binutils/testsuite/binutils-all/pr26160.dwp.bz2 + + # Regenerate crc table in libiberty/crc32.c + cd libiberty + sed -n '/^ #include /,/^ \}$/p' crc32.c > crcgen.c + gcc -o crcgen crcgen.c + sed '/crc_v3\.txt/{n; q}' crc32.c > crc32.c.new + ./crcgen >> crc32.c.new + sed '1,/^};$/d' crc32.c >> crc32.c.new + mv crc32.c.new crc32.c + cd .. + + # bfd-in2.h is required to run autoreconf, but we don't have it yet + cd bfd + cp configure.ac configure.ac.bak + sed -i "s/bfd-in3.h:bfd-in2.h //" configure.ac + AUTOPOINT=true ACLOCAL=aclocal-1.15 AUTOMAKE=automake-1.15 autoreconf-2.69 -fi + ./configure + make headers + mv configure.ac.bak configure.ac + make distclean + cd .. + + # Regenerate top-level (autogen + autotools) + autogen Makefile.def + ACLOCAL=aclocal-1.15 autoreconf-2.69 -fi + + # Regenerate autoconf + for dir in bfd binutils gas gold gprof intl ld libctf libiberty libsframe opcodes; do + cd $dir + ACLOCAL=aclocal-1.15 AUTOMAKE=automake-1.15 autoreconf-2.69 -fi + cd .. + done + cd gprofng + # there is no libtool in gprofng, and libtoolize fails + LIBTOOLIZE=true ACLOCAL=aclocal-1.15 AUTOMAKE=automake-1.15 autoreconf-2.69 -fi + cd .. + + # Rebuild dependencies in libiberty/Makefile.in + cd libiberty + ./configure --enable-maintainer-mode + make maint-deps + make distclean + cd .. + + # Regenerate MeP sections + ./bfd/mep-relocs.pl +} + +src_configure() { + mkdir build + cd build + + LDFLAGS="-static" \ + ../configure \ + --prefix="${KERNEL_SYSROOT}" \ + --libdir="${KERNEL_SYSROOT}/lib" \ + --build="${TARGET}" \ + --host="${TARGET}" \ + --target="${KERNEL_TARGET}" \ + --enable-static \ + --disable-nls \ + --disable-multilib \ + --disable-plugins \ + --disable-gprofng \ + --enable-threads \ + --enable-64-bit-bfd \ + --enable-gold \ + --enable-ld=default \ + --enable-install-libiberty \ + --enable-deterministic-archives \ + --enable-targets=x86_64-elf,i386-elf \ + --with-system-zlib \ + --program-prefix="" \ + --with-sysroot="${KERNEL_SYSROOT}" \ + --srcdir=.. +} + +src_compile() { + make -C opcodes i386-gen + default_src_compile +} + +src_install() { + make "${MAKEJOBS}" install \ + DESTDIR="${DESTDIR}" \ + prefix="${KERNEL_SYSROOT}" \ + libdir="${KERNEL_SYSROOT}/lib" + + # Create triplet symlinks + pushd "${DESTDIR}${KERNEL_SYSROOT}/bin" + for f in *; do + ln -sf "${KERNEL_SYSROOT}/bin/${f}" "${KERNEL_TARGET}-${f}" + done + popd + + # FIXME: Binutils' manpages dates are not reproducible + rm -rf "${DESTDIR}${KERNEL_SYSROOT}/share/man" +} diff --git a/steps-guix/binutils-2.41/patches/new-gettext.patch b/steps-guix/binutils-2.41/patches/new-gettext.patch new file mode 100644 index 00000000..7b2d69cc --- /dev/null +++ b/steps-guix/binutils-2.41/patches/new-gettext.patch @@ -0,0 +1,17 @@ +SPDX-FileCopyrightText: 2023 Samuel Tyler + +SPDX-License-Identifier: GPL-3.0-or-later + +In new gettext external is required for AM_GNU_GETTEXT. + +--- binutils-2.41/intl/configure.ac 2023-02-07 18:57:56.350832016 +1100 ++++ binutils-2.41/intl/configure.ac 2023-02-07 18:58:07.310054484 +1100 +@@ -4,7 +4,7 @@ + AC_CONFIG_HEADER(config.h) + AC_CONFIG_MACRO_DIR(../config) + AM_GNU_GETTEXT_VERSION(0.12.1) +-AM_GNU_GETTEXT([], [need-ngettext]) ++AM_GNU_GETTEXT([external], [need-ngettext]) + + # This replaces the extensive use of DEFS in the original Makefile.in. + AC_DEFINE(IN_LIBINTL, 1, [Define because this is libintl.]) diff --git a/steps-guix/binutils-2.41/patches/no-maint-functions-texi.patch b/steps-guix/binutils-2.41/patches/no-maint-functions-texi.patch new file mode 100644 index 00000000..3f4d1127 --- /dev/null +++ b/steps-guix/binutils-2.41/patches/no-maint-functions-texi.patch @@ -0,0 +1,26 @@ +SPDX-FileCopyrightText: 2023 Samuel Tyler + +SPDX-License-Identifier: GPL-3.0-or-later + +Ensure functions.texi dependencies are satisfied. + +--- binutils-2.41/libiberty/Makefile.in 2023-12-08 15:18:57.985791235 +1100 ++++ binutils-2.41/libiberty/Makefile.in 2023-12-08 15:19:15.391252344 +1100 +@@ -368,12 +368,12 @@ + libiberty.html : $(srcdir)/libiberty.texi $(TEXISRC) + $(MAKEINFO) --no-split --html -I$(srcdir) -o $@ $< + +-@MAINT@$(srcdir)/functions.texi : stamp-functions +-@MAINT@ @true ++$(srcdir)/functions.texi : stamp-functions ++ @true + +-@MAINT@stamp-functions : $(CFILES:%=$(srcdir)/%) $(TEXIFILES:%=$(srcdir)/%) $(srcdir)/gather-docs Makefile +-@MAINT@@HAVE_PERL@ $(PERL) $(srcdir)/gather-docs $(srcdir) $(srcdir)/functions.texi $(CFILES) $(TEXIFILES) +-@MAINT@ echo stamp > stamp-functions ++stamp-functions : $(CFILES:%=$(srcdir)/%) $(TEXIFILES:%=$(srcdir)/%) $(srcdir)/gather-docs Makefile ++@HAVE_PERL@ $(PERL) $(srcdir)/gather-docs $(srcdir) $(srcdir)/functions.texi $(CFILES) $(TEXIFILES) ++ echo stamp > stamp-functions + + INSTALL_DEST = @INSTALL_DEST@ + install: install_to_$(INSTALL_DEST) install-subdir diff --git a/steps-guix/binutils-2.41/patches/no-maint-opcodes.patch b/steps-guix/binutils-2.41/patches/no-maint-opcodes.patch new file mode 100644 index 00000000..d83e36f6 --- /dev/null +++ b/steps-guix/binutils-2.41/patches/no-maint-opcodes.patch @@ -0,0 +1,18 @@ +SPDX-FileCopyrightText: 2022 Dor Askayo +SPDX-FileCopyrightText: 2023 Samuel Tyler + +SPDX-License-Identifier: GPL-3.0-or-later + +Ensure i386-tbl.h dependencies are satisfied. + +--- binutils-2.41/opcodes/Makefile.am 2023-12-08 17:13:05.669136957 +1100 ++++ binutils-2.41/opcodes/Makefile.am 2023-12-08 17:13:18.410480026 +1100 +@@ -540,7 +540,7 @@ + # i386-gen will generate all headers in one go. Use a pattern rule to properly + # express this, with the inner dash ('-') arbitrarily chosen to be the stem. + $(srcdir)/i386%tbl.h $(srcdir)/i386%init.h $(srcdir)/i386%mnem.h: \ +- @MAINT@ i386-gen$(EXEEXT_FOR_BUILD) i386-opc.tbl i386-reg.tbl i386-opc.h ++ i386-gen$(EXEEXT_FOR_BUILD) i386-opc.tbl i386-reg.tbl i386-opc.h + $(AM_V_GEN)$(CPP) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) - \ + < $(srcdir)/i386-opc.tbl \ + | ./i386-gen$(EXEEXT_FOR_BUILD) --srcdir $(srcdir) diff --git a/steps-guix/binutils-2.41/sources b/steps-guix/binutils-2.41/sources new file mode 100644 index 00000000..054c5bed --- /dev/null +++ b/steps-guix/binutils-2.41/sources @@ -0,0 +1 @@ +f https://mirrors.kernel.org/gnu/binutils/binutils-2.41.tar.xz ae9a5789e23459e59606e6714723f2d3ffc31c03174191ef0d015bdf06007450 \ No newline at end of file diff --git a/steps-guix/helpers.sh b/steps-guix/helpers.sh new file mode 100755 index 00000000..b05f195f --- /dev/null +++ b/steps-guix/helpers.sh @@ -0,0 +1,617 @@ +#!/bin/bash -e + +# SPDX-FileCopyrightText: 2021 Andrius Štikonas +# SPDX-FileCopyrightText: 2021-22 Samuel Tyler +# SPDX-FileCopyrightText: 2021 Paul Dersey +# SPDX-FileCopyrightText: 2021 Melg Eight +# +# SPDX-License-Identifier: GPL-3.0-or-later + +# Set constant umask +umask 022 + +# Get a list of files +get_files() { + echo "." + _get_files "${1}" +} + +_get_files() { + local prefix + prefix="${1}" + fs= + if [ -n "$(ls 2>/dev/null)" ]; then + fs=$(echo *) + fi + if [ -n "$(ls .[0-z]* 2>/dev/null)" ]; then + fs="${fs} $(echo .[0-z]*)" + fi + for f in ${fs}; do + # Archive symlinks to directories as symlinks + echo "${prefix}/${f}" + if [ -d "./${f}" ] && ! [ -h "./${f}" ]; then + cd "./${f}" + _get_files "${prefix}/${f}" + cd .. + fi + done +} + +# Reset all timestamps to unix time 0 +reset_timestamp() { + if command -v find >/dev/null 2>&1; then + # find does not error out on exec error + find . -print0 | xargs -0 touch -h -t 197001010000.00 + else + # A rudimentary find implementation that does the trick + fs= + if [ -n "$(ls 2>/dev/null)" ]; then + fs=$(echo ./*) + fi + if [ -n "$(ls .[0-z]* 2>/dev/null)" ]; then + fs="${fs} $(echo .[0-z]*)" + fi + for f in ${fs}; do + touch -h -t 197001010000.00 "./${f}" + if [ -d "./${f}" ]; then + cd "./${f}" + reset_timestamp + cd .. + fi + done + fi +} + +# Fake grep +_grep() { + local text="${1}" + local fname="${2}" + if command -v grep >/dev/null 2>&1; then + grep "${text}" "${fname}" + else + # shellcheck disable=SC2162 + while read line; do + case "${line}" in *"${text}"*) + echo "${line}" ;; + esac + done < "${fname}" + fi +} + +# Useful for perl extensions +get_perl_version() { + perl -v | sed -n -re 's/.*[ (]v([0-9\.]*)[ )].*/\1/p' +} + +get_revision() { + local pkg=$1 + local oldpwd="${PWD}" + cd "/external/repo" + # Get revision (n time this package has been built) + revision=$( (ls -1 "${pkg}"* 2>/dev/null || true) | wc -l | sed 's/ *//g') + cd "${oldpwd}" +} + +# Installs binary packages from an earlier run +# This is useful to speed up development cycle +bin_preseed() { + if [ -d "/external/repo-preseeded" ]; then + get_revision "${pkg}" + cd "/external/repo-preseeded" + test -e "${pkg}_${revision}.tar.bz2" || return 1 + if [ "${UPDATE_CHECKSUMS}" = "True" ] || src_checksum "${pkg}" $((revision)); then + echo "${pkg}: installing prebuilt package." + mv "${pkg}_${revision}.tar.bz2" /external/repo || return 1 + cd "/external/repo" + rm -f /tmp/filelist.txt + src_apply "${pkg}" $((revision)) + cd "${SRCDIR}" + return 0 + fi + fi + return 1 +} + +# Removes either an existing package or file +uninstall() { + local in_fs in_pkg symlinks + while [ $# -gt 0 ]; do + removing="$1" + case "${removing}" in + /*) + # Removing a file + echo "removing file: ${removing}." + rm -f "${removing}" + ;; + *) + echo "${removing}: uninstalling." + local oldpwd="${PWD}" + mkdir -p "/tmp/removing" + cd "/tmp/removing" + get_revision "${removing}" + local filename="/external/repo/${removing}_$((revision-1)).tar.bz2" + # Initial bzip2 built against meslibc has broken pipes + bzip2 -dc "${filename}" | tar -xf - + # reverse to have files before directories + if command -v find >/dev/null 2>&1; then + find . | sort -r > ../filelist + else + get_files . | tac > ../filelist + fi + # shellcheck disable=SC2162 + while read file; do + if [ -d "${file}" ]; then + if [ -z "$(ls -A "/${file}")" ]; then + rmdir "/${file}" + fi + elif [ -h "${file}" ]; then + symlinks="${symlinks} ${file}" + else + # in some cases we might be uninstalling a file that has already been overwritten + # in this case we don't want to remove it + in_fs="$(sha256sum "${file}" 2>/dev/null | cut -d' ' -f1)" + in_pkg="$(sha256sum "/${file}" 2>/dev/null | cut -d' ' -f1)" + if [ "${in_fs}" = "${in_pkg}" ]; then + rm -f "/${file}" + fi + fi + done < ../filelist + rm -f ../filelist + for link in ${symlinks}; do + if [ ! -e "/${link}" ]; then + rm -f "/${link}" + fi + done + cd "${oldpwd}" + rm -rf "/tmp/removing" + ;; + esac + shift + done +} + +# Common build steps +# Build function provides a few common stages with default implementation +# that can be overridden on per package basis in the build script. +# build takes two arguments: +# 1) name-version of the package +# 2) optionally specify build script. Default is pass$((revision+1)).sh +# 3) optionally specify directory to cd into +build() { + pkg=$1 + get_revision "${pkg}" + script_name=${2:-pass$((revision+1)).sh} + dirname=${3:-${pkg}} + + # shellcheck disable=SC2015 + bin_preseed && return || true # Normal build if preseed fails + + cd "${SRCDIR}/${pkg}" || (echo "Cannot cd into ${pkg}!"; kill $$) + echo "${pkg}: beginning build using script ${script_name}" + base_dir="${PWD}" + if [ -e "${base_dir}/patches-$(basename "${script_name}" .sh)" ]; then + patch_dir="${base_dir}/patches-$(basename "${script_name}" .sh)" + else + patch_dir="${base_dir}/patches" + fi + mk_dir="${base_dir}/mk" + files_dir="${base_dir}/files" + + rm -rf "build" + mkdir "build" + cd "build" + + build_script="${base_dir}/${script_name}" + if test -e "${build_script}"; then + # shellcheck source=/dev/null + . "${build_script}" + fi + + echo "${pkg}: getting sources." + build_stage=src_get + call $build_stage + + echo "${pkg}: unpacking source." + build_stage=src_unpack + call $build_stage + unset EXTRA_DISTFILES + + cd "${dirname}" || (echo "Cannot cd into build/${dirname}!"; kill $$) + + echo "${pkg}: preparing source." + build_stage=src_prepare + call $build_stage + + echo "${pkg}: configuring source." + build_stage=src_configure + call $build_stage + + echo "${pkg}: compiling source." + build_stage=src_compile + call $build_stage + + echo "${pkg}: install to fakeroot." + mkdir -p "${DESTDIR}" + build_stage=src_install + call $build_stage + + echo "${pkg}: postprocess binaries." + build_stage=src_postprocess + call $build_stage + + echo "${pkg}: creating package." + cd "${DESTDIR}" + src_pkg + + src_checksum "${pkg}" "${revision}" + + echo "${pkg}: cleaning up." + rm -rf "${SRCDIR}/${pkg}/build" + rm -rf "${DESTDIR}" + + echo "${pkg}: installing package." + src_apply "${pkg}" "${revision}" + + echo "${pkg}: build successful" + + cd "${SRCDIR}" + + unset -f src_get src_unpack src_prepare src_configure src_compile src_install src_postprocess + unset extract +} + +# An inventive way to randomise with what we know we always have +randomize() { + if command -v shuf >/dev/null 2>&1; then + # shellcheck disable=SC2086 + shuf -e ${1} | tr '\n' ' ' + else + mkdir -p /tmp/random + for item in ${1}; do + touch --date=@${RANDOM} /tmp/random/"${item//\//~*~*}" + done + # cannot rely on find existing + # shellcheck disable=SC2012 + ls -1 -t /tmp/random | sed 's:~\*~\*:/:g' | tr '\n' ' ' + rm -r /tmp/random + fi +} + +download_source_line() { + upstream_url="${1}" + checksum="${2}" + fname="${3}" + if ! [ -e "${fname}" ]; then + for mirror in $(randomize "${MIRRORS}"); do + # In qemu SimpleMirror is not running on the guest os, use qemu IP + case "${QEMU}-${mirror}" in 'True-http://127.0.0.1'*) + mirror="http://10.0.2.2${mirror#'http://127.0.0.1'}" + esac + mirror_url="${mirror}/${fname}" + echo "${mirror_url}" + curl --fail --retry 3 --location "${mirror_url}" --output "${fname}" || true && break + done + if ! [ -e "${fname}" ] && [ "${upstream_url}" != "_" ]; then + curl --fail --retry 3 --location "${upstream_url}" --output "${fname}" || true + fi + fi +} + +check_source_line() { + url="${1}" + checksum="${2}" + fname="${3}" + if ! [ -e "${fname}" ]; then + echo "${fname} does not exist!" + false + fi + echo "${checksum} ${fname}" > "${fname}.sum" + sha256sum -c "${fname}.sum" + rm "${fname}.sum" +} + +source_line_action() { + action="$1" + shift + type="$1" + shift + case $type in + "g" | "git") + shift + ;; + esac + url="${1}" + checksum="${2}" + fname="${3}" + # Default to basename of url if not given + fname="${fname:-$(basename "${url}")}" + $action "$url" "$checksum" "$fname" +} + +# Default get function that downloads source tarballs. +default_src_get() { + # shellcheck disable=SC2153 + cd "${DISTFILES}" + # shellcheck disable=SC2162 + while read line; do + # This is intentional - we want to split out ${line} into separate arguments. + # shellcheck disable=SC2086 + source_line_action download_source_line ${line} + done < "${base_dir}/sources" + # shellcheck disable=SC2162 + while read line; do + # This is intentional - we want to split out ${line} into separate arguments. + # shellcheck disable=SC2086 + source_line_action check_source_line ${line} + done < "${base_dir}/sources" + cd - +} + +# Intelligently extracts a file based upon its filetype. +extract_file() { + f="${3:-$(basename "${1}")}" + # shellcheck disable=SC2154 + case "${noextract}" in + *${f}*) + cp "${DISTFILES}/${f}" . + ;; + *) + case "${f}" in + *.tar* | *.tgz) + # shellcheck disable=SC2153 + if test -e "${PREFIX}/libexec/rmt"; then + # Again, we want to split out into words. + # shellcheck disable=SC2086 + tar --no-same-owner -xf "${DISTFILES}/${f}" ${extract} + else + # shellcheck disable=SC2086 + case "${f}" in + *.tar.gz) tar -xzf "${DISTFILES}/${f}" ${extract} ;; + *.tar.bz2) + # Initial bzip2 built against meslibc has broken pipes + bzip2 -dc "${DISTFILES}/${f}" | tar -xf - ${extract} ;; + *.tar.xz | *.tar.lzma) + if test -e "${PREFIX}/bin/xz"; then + tar -xf "${DISTFILES}/${f}" --use-compress-program=xz ${extract} + else + unxz --file "${DISTFILES}/${f}" | tar -xf - ${extract} + fi + ;; + esac + fi + ;; + *) + cp "${DISTFILES}/${f}" . + ;; + esac + ;; + esac +} + +# Default unpacking function that unpacks all sources. +default_src_unpack() { + # Handle the first one differently + first_line=$(head -n 1 ../sources) + # Again, we want to split out into words. + # shellcheck disable=SC2086 + source_line_action extract_file ${first_line} + # This assumes there is only one directory in the tarball + # Get the dirname "smartly" + if ! [ -e "${dirname}" ]; then + for i in *; do + if [ -d "${i}" ]; then + dirname="${i}" + break + fi + done + fi + if ! [ -e "${dirname}" ]; then + # there are no directories extracted + dirname=. + fi + # shellcheck disable=SC2162 + tail -n +2 ../sources | while read line; do + # shellcheck disable=SC2086 + source_line_action extract_file ${line} + done +} + +# Default function to prepare source code. +# It applies all patches from patch_dir (at the moment only -p0 patches are supported). +# Patches are applied from the parent directory. +# Then it copies our custom makefile and any other custom files from files directory. +default_src_prepare() { + if test -d "${patch_dir}"; then + if ls "${patch_dir}"/*.patch >/dev/null 2>&1; then + for p in "${patch_dir}"/*.patch; do + echo "Applying patch: ${p}" + patch -d.. -Np0 < "${p}" + done + fi + fi + + makefile="${mk_dir}/main.mk" + if test -e "${makefile}"; then + cp "${makefile}" Makefile + fi + + if test -d "${files_dir}"; then + cp --no-preserve=mode "${files_dir}"/* "${PWD}/" + fi +} + +# Default function for configuring source. +default_src_configure() { + : +} + +# Default function for compiling source. It simply runs make without any parameters. +default_src_compile() { + make "${MAKEJOBS}" -f Makefile PREFIX="${PREFIX}" +} + +# Default installing function. PREFIX should be set by run.sh script. +# Note that upstream makefiles might ignore PREFIX and have to be configured in configure stage. +default_src_install() { + make -f Makefile install PREFIX="${PREFIX}" DESTDIR="${DESTDIR}" +} + +# Helper function for permissions +_do_strip() { + # shellcheck disable=SC2124 + local f="${@: -1}" + if ! [ -w "${f}" ]; then + local perms + perms="$(stat -c %a "${f}")" + chmod u+w "${f}" + fi + strip "$@" + if [ -n "${perms}" ]; then + chmod "${perms}" "${f}" + fi +} + +# Default function for postprocessing binaries. +default_src_postprocess() { + if (command -v find && command -v file && command -v strip) >/dev/null 2>&1; then + # Logic largely taken from void linux 06-strip-and-debug-pkgs.sh + # shellcheck disable=SC2162 + find "${DESTDIR}" -type f | while read f; do + case "$(file -bi "${f}")" in + application/x-executable*) _do_strip "${f}" ;; + application/x-sharedlib*|application/x-pie-executable*) + machine_set="$(file -b "${f}")" + case "${machine_set}" in + *no\ machine*) ;; # don't strip ELF container-only + *) _do_strip --strip-unneeded "${f}" ;; + esac + ;; + application/x-archive*) _do_strip --strip-debug "${f}" ;; + esac + done + fi +} + +src_pkg() { + touch -t 197001010000.00 . + reset_timestamp + + local tar_basename="${pkg}_${revision}.tar" + local dest_tar="/external/repo/${tar_basename}" + local filelist=/tmp/filelist.txt + + cd /external/repo + # If grep is unavailable, then tar --sort is unavailable. + # So this does not need a command -v grep. + if tar --help | grep ' \-\-sort' >/dev/null 2>&1; then + tar -C "${DESTDIR}" --sort=name --hard-dereference \ + --numeric-owner --owner=0 --group=0 --mode=go=rX,u+rw -cf "${dest_tar}" . + else + local olddir + olddir=$PWD + cd "${DESTDIR}" + local null + if command -v find >/dev/null 2>&1 && command -v sort >/dev/null 2>&1; then + find . -print0 | LC_ALL=C sort -z > "${filelist}" + null="--null" + elif command -v sort >/dev/null 2>&1; then + get_files . | LC_ALL=C sort > "${filelist}" + else + get_files . > ${filelist} + fi + tar --no-recursion ${null} --files-from "${filelist}" \ + --numeric-owner --owner=0 --group=0 --mode=go=rX,u+rw -cf "${dest_tar}" + rm -f "$filelist" + cd "$olddir" + fi + touch -t 197001010000.00 "${tar_basename}" + bzip2 --best "${tar_basename}" +} + +src_checksum() { + local pkg=$1 revision=$2 + local rval=0 + if ! [ "$UPDATE_CHECKSUMS" = True ] ; then + # We avoid using pipes as that is not supported by initial sha256sum from mescc-tools-extra + local checksum_file=/tmp/checksum + _grep "${pkg}_${revision}.tar.bz2" "${SRCDIR}/SHA256SUMS.pkgs" > "${checksum_file}" || true + # Check there is something in checksum_file + if ! [ -s "${checksum_file}" ]; then + echo "${pkg}: no checksum stored!" + false + fi + echo "${pkg}: checksumming created package." + sha256sum -c "${checksum_file}" || rval=$? + rm "${checksum_file}" + fi + return "${rval}" +} + +src_apply() { + local pkg="${1}" revision="${2}" + local TAR_PREFIX BZIP2_PREFIX + + # Make sure we have at least one copy of tar + if [[ "${pkg}" == tar-* ]]; then + mkdir -p /tmp + cp "${PREFIX}/bin/tar" "/tmp/tar" + TAR_PREFIX="/tmp/" + fi + + # Bash does not like to be overwritten + if [[ "${pkg}" == bash-* ]]; then + rm "${PREFIX}/bin/bash" + fi + + # Overwriting files is mega busted, so do it manually + # shellcheck disable=SC2162 + if [ -e /tmp/filelist.txt ]; then + while IFS= read -d $'\0' file; do + rm -f "/${file}" >/dev/null 2>&1 || true + done < /tmp/filelist.txt + fi + + # Bzip2 does not like to be overwritten + if [[ "${pkg}" == bzip2-* ]]; then + mkdir -p /tmp + mv "${PREFIX}/bin/bzip2" "/tmp/bzip2" + BZIP2_PREFIX="/tmp/" + fi + "${BZIP2_PREFIX}bzip2" -dc "/external/repo/${pkg}_${revision}.tar.bz2" | \ + "${TAR_PREFIX}tar" -C / -xpf - + rm -f "/tmp/bzip2" "/tmp/tar" +} + +# Check if bash function exists +fn_exists() { + test "$(type -t "$1")" == 'function' +} + +# Call package specific function or default implementation. +call() { + if fn_exists "$1"; then + $1 + else + default_"${1}" + fi +} + +# Call default build stage function +default() { + "default_${build_stage}" +} + +# Reusable helpers for sysroot-oriented stages (e.g. kernel toolchain/kernel build). +sysroot_stage_init() { + : "${KERNEL_SYSROOT:=/kernel-toolchain}" +} + +sysroot_src_install_default() { + # Generic install helper for packages that should install into a dedicated sysroot. + # Package-specific post-install actions stay in each package script. + sysroot_stage_init + make "${MAKEJOBS}" install \ + DESTDIR="${DESTDIR}" \ + prefix="${KERNEL_SYSROOT}" \ + libdir="${KERNEL_SYSROOT}/lib" +} diff --git a/steps-guix/manifest b/steps-guix/manifest new file mode 100644 index 00000000..126e82dc --- /dev/null +++ b/steps-guix/manifest @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Guix extension manifest. Runs after main /steps manifest when BUILD_GUIX_ALSO=True. + +# We need a 64-bit kernel to enable Guix to run 64-bit programs. +build: binutils-2.41 + diff --git a/steps/improve/after.sh b/steps/improve/after.sh index 727b29e8..f637a8ca 100644 --- a/steps/improve/after.sh +++ b/steps/improve/after.sh @@ -27,7 +27,7 @@ if [ "${BUILD_GUIX_ALSO}" = True ]; then echo 'BUILD_GUIX_ALSO=False' >> /steps/bootstrap.cfg /script-generator /steps-guix/manifest /steps - kaem --file /steps-guix/0.sh + bash /steps-guix/0.sh fi if [ "${INTERACTIVE}" = True ]; then diff --git a/steps/improve/make_bootable.sh b/steps/improve/make_bootable.sh index b0535f11..22195373 100644 --- a/steps/improve/make_bootable.sh +++ b/steps/improve/make_bootable.sh @@ -87,7 +87,32 @@ fi if [ "${CHROOT}" = False ]; then dhcpcd --waitip=4 fi +EOF +if [ "${BUILD_GUIX_ALSO}" = True ]; then +cat >> /init <<- 'EOF' +run_steps_guix_if_requested() { + if [ "${BUILD_GUIX_ALSO}" != True ]; then + return 1 + fi + if [ ! -f /steps-guix/manifest ]; then + echo "BUILD_GUIX_ALSO is True but /steps-guix/manifest is missing." >&2 + exit 1 + fi + + sed -i '/^BUILD_GUIX_ALSO=/d' /steps/bootstrap.cfg + echo 'BUILD_GUIX_ALSO=False' >> /steps/bootstrap.cfg + + /script-generator /steps-guix/manifest /steps + bash /steps-guix/0.sh + return 0 +} + +run_steps_guix_if_requested || true +EOF +fi + +cat >> /init <<- 'EOF' if [ "${QEMU}" = True ] && [ "${BARE_METAL}" = False ]; then if [ -c /dev/ttyS0 ]; then env - PATH=${PREFIX}/bin PS1="\w # " bash -i /dev/ttyS0 2>&1 diff --git a/steps/improve/reconfigure.sh b/steps/improve/reconfigure.sh index a7ce0260..cb30756d 100644 --- a/steps/improve/reconfigure.sh +++ b/steps/improve/reconfigure.sh @@ -3,4 +3,4 @@ # SPDX-License-Identifier: GPL-3.0-or-later set -ex /configurator /steps/configurator -/script-generator /steps/manifest /steps +/script-generator /steps/manifest