script-generator: support explicit manifest/config roots and wire all callers

lib/generator.py

@@ -27,14 +27,18 @@ class Generator():
     distfiles_dir = os.path.join(git_dir, 'distfiles')
 
     # pylint: disable=too-many-arguments,too-many-positional-arguments
-    def __init__(self, arch, external_sources, early_preseed, repo_path, mirrors):
+    def __init__(self, arch, external_sources, early_preseed, repo_path, mirrors,
+                 build_guix_also=False):
         self.arch = arch
         self.early_preseed = early_preseed
         self.external_sources = external_sources
         self.repo_path = repo_path
         self.mirrors = mirrors
-        self.source_manifest = self.get_source_manifest(not self.external_sources)
+        self.build_guix_also = build_guix_also
-        self.early_source_manifest = self.get_source_manifest(True)
+        self.source_manifest = self.get_source_manifest(not self.external_sources,
+                                                        build_guix_also=self.build_guix_also)
+        self.early_source_manifest = self.get_source_manifest(True,
+                                                              build_guix_also=self.build_guix_also)
         self.target_dir = None
         self.external_dir = None
 
@@ -117,6 +121,13 @@ class Generator():
         self.get_packages()
 
         shutil.copytree(os.path.join(self.git_dir, 'steps'), os.path.join(self.target_dir, 'steps'))
+        if self.build_guix_also:
+            steps_guix_dir = os.path.join(self.git_dir, 'steps-guix')
+            if not os.path.isdir(steps_guix_dir):
+                raise ValueError("steps-guix directory does not exist while --build-guix-also is set.")
+            if not os.path.isfile(os.path.join(steps_guix_dir, 'manifest')):
+                raise ValueError("steps-guix/manifest does not exist while --build-guix-also is set.")
+            shutil.copytree(steps_guix_dir, os.path.join(self.target_dir, 'steps-guix'))
 
     def stage0_posix(self, kernel_bootstrap=False):
         """Copy in all of the stage0-posix"""
@@ -344,43 +355,55 @@ this script the next time")
             self.check_file(path, line[0])
 
     @classmethod
-    def get_source_manifest(cls, pre_network=False):
+    def get_source_manifest(cls, pre_network=False, build_guix_also=False):
         """
         Generate a source manifest for the system.
         """
         entries = []
         directory = os.path.relpath(cls.distfiles_dir, cls.git_dir)
 
-        # Find all source files
+        manifests = [os.path.join(cls.git_dir, 'steps')]
-        steps_dir = os.path.join(cls.git_dir, 'steps')
+        if build_guix_also:
-        with open(os.path.join(steps_dir, 'manifest'), 'r', encoding="utf_8") as file:
+            steps_guix_dir = os.path.join(cls.git_dir, 'steps-guix')
-            for line in file:
+            if not os.path.isdir(steps_guix_dir):
-                if pre_network and line.strip().startswith("improve: ") and "network" in line:
+                raise ValueError("steps-guix directory does not exist while --build-guix-also is set.")
-                    break
+            manifests.append(steps_guix_dir)
 
-                if not line.strip().startswith("build: "):
+        for steps_dir in manifests:
-                    continue
+            manifest_path = os.path.join(steps_dir, 'manifest')
+            if not os.path.isfile(manifest_path):
+                if steps_dir.endswith('steps-guix'):
+                    raise ValueError("steps-guix/manifest does not exist while --build-guix-also is set.")
+                raise ValueError(f"Missing manifest: {manifest_path}")
 
-                step = line.split(" ")[1].split("#")[0].strip()
+            with open(manifest_path, 'r', encoding="utf_8") as file:
-                sourcef = os.path.join(steps_dir, step, "sources")
+                for line in file:
-                if os.path.exists(sourcef):
+                    if pre_network and line.strip().startswith("improve: ") and "network" in line:
-                    # Read sources from the source file
+                        break
-                    with open(sourcef, "r", encoding="utf_8") as sources:
-                        for source in sources.readlines():
-                            source = source.strip().split(" ")
 
-                            if source[0] == "g" or source[0] == "git":
+                    if not line.strip().startswith("build: "):
-                                source[1:] = source[2:]
+                        continue
 
-                            if len(source) > 3:
+                    step = line.split(" ")[1].split("#")[0].strip()
-                                file_name = source[3]
+                    sourcef = os.path.join(steps_dir, step, "sources")
-                            else:
+                    if os.path.exists(sourcef):
-                                # Automatically determine file name based on URL.
+                        # Read sources from the source file
-                                file_name = os.path.basename(source[1])
+                        with open(sourcef, "r", encoding="utf_8") as sources:
+                            for source in sources.readlines():
+                                source = source.strip().split(" ")
 
-                            entry = (source[2], directory, source[1], file_name)
+                                if source[0] == "g" or source[0] == "git":
-                            if entry not in entries:
+                                    source[1:] = source[2:]
-                                entries.append(entry)
+
+                                if len(source) > 3:
+                                    file_name = source[3]
+                                else:
+                                    # Automatically determine file name based on URL.
+                                    file_name = os.path.basename(source[1])
+
+                                entry = (source[2], directory, source[1], file_name)
+                                if entry not in entries:
+                                    entries.append(entry)
 
         return entries
 

rootfs.py

@@ -43,6 +43,7 @@ def create_configuration_file(args):
         config.write(f"INTERACTIVE={args.interactive}\n")
         config.write(f"QEMU={args.qemu}\n")
         config.write(f"BARE_METAL={args.bare_metal or (args.qemu and args.interactive)}\n")
+        config.write(f"BUILD_GUIX_ALSO={args.build_guix_also}\n")
         if (args.bare_metal or args.qemu) and not args.kernel:
             if args.repo or args.external_sources:
                 config.write("DISK=sdb1\n")
@@ -95,6 +96,9 @@ def main():
     parser.add_argument("--build-kernels",
                         help="Also build kernels in chroot and bwrap builds",
                         action="store_true")
+    parser.add_argument("--build-guix-also",
+                        help="After main steps finish, switch to steps-guix and run its manifest",
+                        action="store_true")
     parser.add_argument("--no-create-config",
                         help="Do not automatically create config file",
                         action="store_true")
@@ -227,7 +231,8 @@ def main():
                           external_sources=args.external_sources,
                           repo_path=args.repo,
                           early_preseed=args.early_preseed,
-                          mirrors=args.mirrors)
+                          mirrors=args.mirrors,
+                          build_guix_also=args.build_guix_also)
 
     bootstrap(args, generator, target, args.target_size, cleanup)
     cleanup()

seed/preseeded.kaem

@@ -4,5 +4,5 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-/script-generator /steps/manifest
+/script-generator /steps/manifest /steps
 /usr/bin/kaem --file /preseed-jump.kaem

seed/script-generator.c

@@ -32,6 +32,48 @@ struct Directive {
 };
 typedef struct Directive Directive;
 
+char *steps_root = "/steps";
+char *config_root = "/steps";
+
+char *join_path(const char *base, const char *suffix) {
+	char *out = calloc(MAX_STRING, sizeof(char));
+	strcpy(out, base);
+	if (strlen(out) > 0 && out[strlen(out) - 1] != '/') {
+		strcat(out, "/");
+	}
+	while (*suffix == '/') {
+		suffix += 1;
+	}
+	strcat(out, suffix);
+	return out;
+}
+
+char *config_path(const char *suffix) {
+	return join_path(config_root, suffix);
+}
+
+char *dirname_from_path(const char *path) {
+	char *slash = strrchr(path, '/');
+	char *out = calloc(MAX_STRING, sizeof(char));
+	if (slash == NULL) {
+		strcpy(out, ".");
+		return out;
+	}
+	if (slash == path) {
+		strcpy(out, "/");
+		return out;
+	}
+	strncpy(out, path, slash - path);
+	return out;
+}
+
+void write_steps_prefix(FILE *out) {
+	fputs(steps_root, out);
+	if (steps_root[strlen(steps_root) - 1] != '/') {
+		fputs("/", out);
+	}
+}
+
 /* Tokenizer. */
 
 /* Skip over a comment. */
@@ -120,10 +162,11 @@ typedef struct Variable Variable;
 Variable *variables;
 
 Variable *load_config() {
-	FILE *config = fopen("/steps/bootstrap.cfg", "r");
+	char *config_file = config_path("bootstrap.cfg");
-	/* File does not exist check. */
+	FILE *config = fopen(config_file, "r");
 	if (config == NULL) {
-		return NULL;
+		fputs("Unable to open bootstrap.cfg\n", stderr);
+		exit(1);
 	}
 
 	char *line = calloc(MAX_STRING, sizeof(char));
@@ -392,9 +435,12 @@ Directive *interpreter(Directive *directives) {
 
 /* Script generator. */
 FILE *start_script(int id, int bash_build) {
-	/* Create the file /steps/$id.sh */
+	/* Create the file ${steps_root}/$id.sh */
 	char *filename = calloc(MAX_STRING, sizeof(char));
-	strcpy(filename, "/steps/");
+	strcpy(filename, steps_root);
+	if (filename[strlen(filename) - 1] != '/') {
+		strcat(filename, "/");
+	}
 	strcat(filename, int2str(id, 10, 0));
 	strcat(filename, ".sh");
 
@@ -407,6 +453,9 @@ FILE *start_script(int id, int bash_build) {
 	}
 
 	if (bash_build) {
+		char *bootstrap_file = config_path("bootstrap.cfg");
+		char *env_file = config_path("env");
+		char *helpers_file = config_path("helpers.sh");
 		fputs("#!/bin/bash\n", out);
 		if (strcmp(get_var("INTERACTIVE"), "True") == 0) {
 			if (bash_build != 1) {
@@ -420,15 +469,30 @@ FILE *start_script(int id, int bash_build) {
 		} else {
 			fputs("set -e\n", out);
 		}
-		fputs("cd /steps\n", out);
+		fputs("cd ", out);
-		fputs(". ./bootstrap.cfg\n", out);
+		fputs(steps_root, out);
-		fputs(". ./env\n", out);
+		fputs("\n", out);
-		fputs(". ./helpers.sh\n", out);
+		fputs(". ", out);
+		fputs(bootstrap_file, out);
+		fputs("\n", out);
+		fputs(". ", out);
+		fputs(env_file, out);
+		fputs("\n", out);
+		fputs(". ", out);
+		fputs(helpers_file, out);
+		fputs("\n", out);
 	} else {
+		char *env_file = config_path("env");
 		fputs("set -ex\n", out);
-		fputs("cd /steps\n", out);
+		fputs("cd ", out);
+		fputs(steps_root, out);
+		fputs("\n", out);
 		output_config(out);
-		FILE *env = fopen("/steps/env", "r");
+		FILE *env = fopen(env_file, "r");
+		if (env == NULL) {
+			fputs("Unable to open env\n", stderr);
+			exit(1);
+		}
 		char *line = calloc(MAX_STRING, sizeof(char));
 		while (fgets(line, MAX_STRING, env) != 0) {
 			/* Weird M2-Planet behaviour. */
@@ -454,7 +518,7 @@ void output_call_script(FILE *out, char *type, char *name, int bash_build, int s
 	} else {
 		fputs("kaem --file ", out);
 	}
-	fputs("/steps/", out);
+	write_steps_prefix(out);
 	if (strlen(type) != 0) {
 		fputs(type, out);
 		fputs("/", out);
@@ -486,7 +550,8 @@ void generate_preseed_jump(int id) {
 	FILE *out = fopen("/preseed-jump.kaem", "w");
 	fputs("set -ex\n", out);
 	fputs("PATH=/usr/bin\n", out);
-	fputs("bash /steps/", out);
+	fputs("bash ", out);
+	write_steps_prefix(out);
 	fputs(int2str(id, 10, 0), out);
 	fputs(".sh\n", out);
 	fclose(out);
@@ -598,8 +663,8 @@ void generate(Directive *directives) {
 }
 
 void main(int argc, char **argv) {
-	if (argc != 2) {
+	if (argc != 2 && argc != 3) {
-		fputs("Usage: script-generator <script>\n", stderr);
+		fputs("Usage: script-generator <script> [config-root]\n", stderr);
 		exit(1);
 	}
 
@@ -608,12 +673,15 @@ void main(int argc, char **argv) {
 		fputs("Error opening input file\n", stderr);
 		exit(1);
 	}
+	steps_root = dirname_from_path(argv[1]);
+	config_root = (argc == 3) ? argv[2] : steps_root;
 	Directive *directives = tokenizer(in);
 	fclose(in);
 	load_config();
 	directives = interpreter(directives);
 	generate(directives);
-	FILE *config = fopen("/steps/bootstrap.cfg", "w");
+	char *config_file = config_path("bootstrap.cfg");
+	FILE *config = fopen(config_file, "w");
 	output_config(config);
 	fclose(config);
 }

seed/seed.kaem

@@ -80,6 +80,6 @@ if match x${UPDATE_CHECKSUMS} xTrue; then
 else
     sha256sum -c script-generator.${ARCH}.checksums
 fi
-./script-generator /steps/manifest
+./script-generator /steps/manifest /steps
 
 kaem --file /steps/0.sh

steps/improve/after.sh

@@ -17,6 +17,19 @@ if [ -d /steps/after ]; then
     done
 fi
 
+if [ "${BUILD_GUIX_ALSO}" = True ]; then
+    if [ ! -f /steps-guix/manifest ]; then
+        echo "BUILD_GUIX_ALSO is True but /steps-guix/manifest is missing." >&2
+        exit 1
+    fi
+
+    sed -i '/^BUILD_GUIX_ALSO=/d' /steps/bootstrap.cfg
+    echo 'BUILD_GUIX_ALSO=False' >> /steps/bootstrap.cfg
+
+    /script-generator /steps-guix/manifest /steps
+    kaem --file /steps-guix/0.sh
+fi
+
 if [ "${INTERACTIVE}" = True ]; then
     env - PATH=${PREFIX}/bin PS1="\w # " setsid openvt -fec1 -- bash -i
 fi

steps/improve/make_bootable.sh

@@ -39,12 +39,46 @@ cd /steps
 . ./env
 . ./helpers.sh
 trap 'env PATH=${PREFIX}/bin PS1="[TRAP] \w # " bash -i' ERR
-# /dev is automounted by the kernel at this point
+
-mount | grep '/proc' &> /dev/null || (mkdir -p /proc; mount -t proc proc /proc)
+setup_kernel_devices() {
-mount | grep '/sys' &> /dev/null || (mkdir -p /sys; mount -t sysfs sysfs /sys)
+    mount | grep ' on /dev ' &> /dev/null || (mkdir -p /dev; mount -t devtmpfs devtmpfs /dev)
-# Make /tmp a ramdisk (speeds up configure etc significantly)
+    mount | grep ' on /proc ' &> /dev/null || (mkdir -p /proc; mount -t proc proc /proc)
-mount | grep '/tmp' &> /dev/null || (mkdir -p /tmp; mount -t tmpfs tmpfs /tmp)
+    mount | grep ' on /sys ' &> /dev/null || (mkdir -p /sys; mount -t sysfs sysfs /sys)
-mount | grep '/dev/shm' &> /dev/null || (mkdir -p /dev/shm; mount -t tmpfs tmpfs /dev/shm)
+    # Make /tmp a ramdisk (speeds up configure etc significantly)
+    mount | grep ' on /tmp ' &> /dev/null || (mkdir -p /tmp; mount -t tmpfs tmpfs /tmp)
+    mount | grep ' on /dev/pts ' &> /dev/null || (mkdir -p /dev/pts; mount -t devpts devpts /dev/pts)
+    mount | grep ' on /dev/shm ' &> /dev/null || (mkdir -p /dev/shm; mount -t tmpfs tmpfs /dev/shm)
+
+    test -c /dev/console || mknod -m 666 /dev/console c 5 1
+    test -c /dev/tty || mknod -m 666 /dev/tty c 5 0
+    test -c /dev/ptmx || mknod -m 666 /dev/ptmx c 5 2
+    test -c /dev/tty0 || mknod -m 666 /dev/tty0 c 4 0
+    test -c /dev/tty1 || mknod -m 666 /dev/tty1 c 4 1
+    test -c /dev/tty2 || mknod -m 666 /dev/tty2 c 4 2
+    test -c /dev/ttyS0 || mknod -m 666 /dev/ttyS0 c 4 64
+}
+
+verify_kernel_devices() {
+    mount | grep ' on /dev ' &> /dev/null &&
+    mount | grep ' on /proc ' &> /dev/null &&
+    mount | grep ' on /sys ' &> /dev/null &&
+    mount | grep ' on /tmp ' &> /dev/null &&
+    mount | grep ' on /dev/pts ' &> /dev/null &&
+    mount | grep ' on /dev/shm ' &> /dev/null &&
+    test -c /dev/console &&
+    test -c /dev/tty &&
+    test -c /dev/ptmx &&
+    test -c /dev/tty0 &&
+    test -c /dev/tty1 &&
+    test -c /dev/tty2 &&
+    test -c /dev/ttyS0
+}
+
+setup_kernel_devices
+verify_kernel_devices || {
+    echo "Kernel device setup verification failed." >&2
+    exit 1
+}
 
 if test -f /swapfile; then
     swapon /swapfile
@@ -54,7 +88,15 @@ if [ "${CHROOT}" = False ]; then
     dhcpcd --waitip=4
 fi
 
-env - PATH=${PREFIX}/bin PS1="\w # " setsid openvt -fec1 -- bash -i
+if [ "${QEMU}" = True ] && [ "${BARE_METAL}" = False ]; then
+    if [ -c /dev/ttyS0 ]; then
+        env - PATH=${PREFIX}/bin PS1="\w # " bash -i </dev/ttyS0 >/dev/ttyS0 2>&1
+    else
+        env - PATH=${PREFIX}/bin PS1="\w # " bash -i </dev/console >/dev/console 2>&1
+    fi
+else
+    env - PATH=${PREFIX}/bin PS1="\w # " setsid openvt -fec1 -- bash -i
+fi
 
 # ignore errors due to fstab or swapfile not existing
 swapoff -a &> /dev/null || true

steps/improve/reconfigure.sh

@@ -3,4 +3,4 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 set -ex
 /configurator /steps/configurator
-/script-generator /steps/manifest
+/script-generator /steps/manifest /steps