diff --git a/lib/generator.py b/lib/generator.py index b3de7f3a..7936b4e5 100755 --- a/lib/generator.py +++ b/lib/generator.py @@ -12,6 +12,7 @@ import hashlib import os import random import shutil +import struct import tarfile import traceback @@ -25,6 +26,7 @@ class Generator(): git_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..') distfiles_dir = os.path.join(git_dir, 'distfiles') + payload_magic = b'LBPAYLD1' # pylint: disable=too-many-arguments,too-many-positional-arguments def __init__(self, arch, external_sources, early_preseed, repo_path, mirrors, @@ -39,6 +41,9 @@ class Generator(): build_guix_also=self.build_guix_also) self.early_source_manifest = self.get_source_manifest(True, build_guix_also=self.build_guix_also) + self.bootstrap_source_manifest = self.source_manifest + self.payload_source_manifest = [] + self.payload_image = None self.target_dir = None self.external_dir = None @@ -50,6 +55,69 @@ class Generator(): self.external_dir = os.path.join(self.target_dir, 'external') self.distfiles() + def _prepare_kernel_bootstrap_payload_manifests(self): + """ + Split early source payload from full offline payload. + """ + # Keep the early builder payload small enough to avoid overrunning + # builder-hex0 memory file allocation before we can jump into Fiwix. + self.bootstrap_source_manifest = self.get_source_manifest(True, build_guix_also=False) + + full_manifest = self.get_source_manifest(False, build_guix_also=self.build_guix_also) + bootstrap_set = set(self.bootstrap_source_manifest) + self.payload_source_manifest = [entry for entry in full_manifest if entry not in bootstrap_set] + + def _copy_manifest_distfiles(self, out_dir, manifest): + os.makedirs(out_dir, exist_ok=True) + for entry in manifest: + file_name = entry[3].strip() + shutil.copy2(os.path.join(self.distfiles_dir, file_name), + os.path.join(out_dir, file_name)) + + def _ensure_manifest_distfiles(self, manifest): + for entry in manifest: + checksum, directory, url, file_name = entry + distfile_path = os.path.join(directory, file_name) + if not os.path.isfile(distfile_path): + self.download_file(url, directory, file_name) + self.check_file(distfile_path, checksum) + + def _create_raw_payload_image(self, target_path, manifest): + if not manifest: + return None + + # Guarantee all payload distfiles exist and match checksums. + self._ensure_manifest_distfiles(manifest) + + files_by_name = {} + for checksum, _, _, file_name in manifest: + if file_name in files_by_name and files_by_name[file_name] != checksum: + raise ValueError(f"Conflicting payload file with same name but different hash: {file_name}") + files_by_name[file_name] = checksum + + payload_path = os.path.join(target_path, "payload.img") + ordered_names = sorted(files_by_name.keys()) + with open(payload_path, "wb") as payload: + payload.write(self.payload_magic) + payload.write(struct.pack(" 0xFFFFFFFF: + raise ValueError(f"Payload file name too long: {file_name}") + + src_path = os.path.join(self.distfiles_dir, file_name) + file_size = os.path.getsize(src_path) + if file_size > 0xFFFFFFFF: + raise ValueError(f"Payload file too large for raw container format: {file_name}") + + payload.write(struct.pack(" (unsigned long) fiwix_len) { - printf("kexec-fiwix: invalid ELF header table (offset=0x%x num=%u entsize=%u)\n", - e_phoff, e_phnum, e_phentsize); - return EXIT_FAILURE; - } /* Load the kernel */ puts("kexec-fiwix: Placing kernel in memory..."); int header_num; - int adjusted_entry = 0; - unsigned int e_entry_phys = e_entry; for (header_num = 0; header_num < e_phnum; header_num++) { char * fiwix_prog_header = &fiwix_mem[e_phoff + header_num * e_phentsize]; - unsigned int p_type = *((unsigned int *) (&fiwix_prog_header[0x00])); unsigned int p_offset = *((unsigned int *) (&fiwix_prog_header[0x04])); unsigned int p_vaddr = *((unsigned int *) (&fiwix_prog_header[0x08])); unsigned int p_paddr = *((unsigned int *) (&fiwix_prog_header[0x0C])); unsigned int p_filesz = *((unsigned int *) (&fiwix_prog_header[0x10])); unsigned int p_memsz = *((unsigned int *) (&fiwix_prog_header[0x14])); - if (p_type != ELF_PT_LOAD) { - continue; - } - if (p_filesz > p_memsz) { - printf("kexec-fiwix: invalid segment %d, p_filesz > p_memsz\n", header_num); - return EXIT_FAILURE; - } - if ((unsigned long) p_offset + (unsigned long) p_filesz > (unsigned long) fiwix_len) { - printf("kexec-fiwix: invalid segment %d, out-of-bounds file range\n", header_num); - return EXIT_FAILURE; - } - if (!adjusted_entry) { - e_entry_phys -= (p_vaddr - p_paddr); - adjusted_entry = 1; + if (header_num == 0) { + e_entry -= (p_vaddr - p_paddr); + printf("ELF physical entry point : 0x%x\n", e_entry); } printf("header %d:\n", header_num); @@ -148,11 +128,6 @@ int main(int argc, char **argv) { memset((void *)p_paddr, 0, p_memsz + 0x10000); memcpy((void *)p_paddr, &fiwix_mem[p_offset], p_filesz); } - if (!adjusted_entry) { - printf("kexec-fiwix: no PT_LOAD segments found in kernel ELF\n"); - return EXIT_FAILURE; - } - printf("ELF physical entry point : 0x%x\n", e_entry_phys); puts("Preparing multiboot info for kernel..."); @@ -181,26 +156,16 @@ int main(int argc, char **argv) { } int filenum; - int found_image = 0; unsigned int filename_addr; for (filenum = 4, filename_addr = 0x201000; filenum <= 14335; filenum++, filename_addr += 1024) { if (!strcmp((char *) filename_addr, initrd_filename)) { printf("Found image at filenum %d\n", filenum); - found_image = 1; break; } } - if (!found_image) { - printf("kexec-fiwix: initrd image not found in file table: %s\n", initrd_filename); - return EXIT_FAILURE; - } unsigned int initrd_src = *((unsigned int *) (0x01000000 + (16 * filenum) + 4)); unsigned int initrd_len = *((unsigned int *) (0x01000000 + (16 * filenum) + 8)); - if (initrd_src == 0 || initrd_len == 0) { - printf("kexec-fiwix: invalid initrd metadata src=0x%08x len=0x%08x\n", initrd_src, initrd_len); - return EXIT_FAILURE; - } printf("initrd_src: 0x%08x\n", initrd_src); printf("initrd_len: 0x%08x\n", initrd_len); @@ -276,6 +241,7 @@ int main(int argc, char **argv) { /* Jump to kernel entry point */ unsigned int magic = MULTIBOOT_BOOTLOADER_MAGIC; + unsigned int dummy = 0; unsigned int multiboot_info_num = (unsigned int) pmultiboot_info; printf("Preparing trampoline...\n"); @@ -294,7 +260,7 @@ int main(int argc, char **argv) { 0xF3, 0xA4, /* rep movsb */ 0xB8, 0x00, 0x00, 0x00, 0x00, /* mov eax, 0x00000000 */ 0xBB, 0x00, 0x00, 0x00, 0x00, /* mov ebx, 0x00000000 */ - 0xFA, /* cli */ + 0xFB, /* sti */ 0xEA, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00 /* jmp far 0x0008:0x00000000 */ }; @@ -304,7 +270,7 @@ int main(int argc, char **argv) { *((unsigned int *) &trampoline[11]) = initrd_len; *((unsigned int *) &trampoline[19]) = magic; *((unsigned int *) &trampoline[24]) = multiboot_info_num; - *((unsigned int *) &trampoline[30]) = e_entry_phys; + *((unsigned int *) &trampoline[30]) = e_entry; memcpy((void *)0x4000, trampoline, sizeof(trampoline)); diff --git a/steps/manifest b/steps/manifest index 9fd6bd7a..d6b7b4b0 100644 --- a/steps/manifest +++ b/steps/manifest @@ -40,7 +40,9 @@ define: BUILD_FIWIX = ( KERNEL_BOOTSTRAP == True || BUILD_KERNELS == True ) build: fiwix-1.5.0-lb1 ( BUILD_FIWIX == True ) build: lwext4-1.0.0-lb1 ( BUILD_FIWIX == True ) build: kexec-fiwix-1.0 ( BUILD_FIWIX == True ) +build: payload-import-1.0 ( KERNEL_BOOTSTRAP == True ) jump: fiwix ( KERNEL_BOOTSTRAP == True ) +improve: import_payload ( KERNEL_BOOTSTRAP == True ) improve: reconfigure ( CONFIGURATOR != True ) define: JOBS = 1 ( KERNEL_BOOTSTRAP == True ) build: make-3.82 diff --git a/steps/payload-import-1.0/pass1.kaem b/steps/payload-import-1.0/pass1.kaem new file mode 100644 index 00000000..fd5d7cac --- /dev/null +++ b/steps/payload-import-1.0/pass1.kaem @@ -0,0 +1,19 @@ +#!/bin/sh +# +# SPDX-FileCopyrightText: 2026 live-bootstrap contributors +# SPDX-License-Identifier: MIT + +set -ex + +cd src +tcc -m32 -march=i386 -std=c89 -I../../tcc/tcc-0.9.27/include -o ${BINDIR}/payload-import payload-import.c +cd .. + +if match x${UPDATE_CHECKSUMS} xTrue; then + sha256sum -o ${pkg}.checksums \ + /usr/bin/payload-import + + cp ${pkg}.checksums ${SRCDIR} +elif test -f ${pkg}.checksums; then + sha256sum -c ${pkg}.checksums +fi diff --git a/steps/payload-import-1.0/src/payload-import.c b/steps/payload-import-1.0/src/payload-import.c new file mode 100644 index 00000000..8be94df3 --- /dev/null +++ b/steps/payload-import-1.0/src/payload-import.c @@ -0,0 +1,297 @@ +/* SPDX-FileCopyrightText: 2026 live-bootstrap contributors */ +/* SPDX-License-Identifier: MIT */ + +#include +#include +#include +#include +#include +#include + +#define MAGIC "LBPAYLD1" +#define MAGIC_LEN 8 +#define MAX_NAME_LEN 1024 +#define COPY_BUFSZ 65536 + +static unsigned int read_u32le(const unsigned char *buf) +{ + return (unsigned int)buf[0] + | ((unsigned int)buf[1] << 8) + | ((unsigned int)buf[2] << 16) + | ((unsigned int)buf[3] << 24); +} + +static int read_exact(FILE *in, void *buf, unsigned int len) +{ + unsigned int got = 0; + unsigned char *out = (unsigned char *)buf; + + while (got < len) { + size_t n = fread(out + got, 1, len - got, in); + if (n == 0) { + return -1; + } + got += (unsigned int)n; + } + return 0; +} + +static int copy_exact(FILE *in, FILE *out, unsigned int len) +{ + unsigned char *buf; + unsigned int remaining = len; + + buf = (unsigned char *)malloc(COPY_BUFSZ); + if (buf == NULL) { + fputs("payload-import: out of memory\n", stderr); + return 1; + } + + while (remaining > 0) { + unsigned int chunk = remaining; + size_t written; + if (chunk > COPY_BUFSZ) { + chunk = COPY_BUFSZ; + } + if (read_exact(in, buf, chunk) != 0) { + free(buf); + return 1; + } + written = fwrite(buf, 1, chunk, out); + if (written != chunk) { + free(buf); + return 1; + } + remaining -= chunk; + } + + free(buf); + return 0; +} + +static int is_valid_name(const char *name) +{ + const unsigned char *s = (const unsigned char *)name; + + if (*s == 0) { + return 0; + } + + while (*s != 0) { + if (*s == '/' || *s == '\\') { + return 0; + } + s += 1; + } + return 1; +} + +static int has_payload_magic(const char *path) +{ + FILE *in; + char magic[MAGIC_LEN]; + + in = fopen(path, "rb"); + if (in == NULL) { + return 1; + } + if (read_exact(in, magic, MAGIC_LEN) != 0) { + fclose(in); + return 1; + } + fclose(in); + if (memcmp(magic, MAGIC, MAGIC_LEN) != 0) { + return 1; + } + return 0; +} + +static int extract_payload(const char *device, const char *dest_dir) +{ + FILE *in; + char magic[MAGIC_LEN]; + unsigned char u32buf[4]; + unsigned int file_count; + unsigned int i; + + in = fopen(device, "rb"); + if (in == NULL) { + fprintf(stderr, "payload-import: cannot open %s: %s\n", device, strerror(errno)); + return 1; + } + + if (read_exact(in, magic, MAGIC_LEN) != 0 || memcmp(magic, MAGIC, MAGIC_LEN) != 0) { + fclose(in); + fprintf(stderr, "payload-import: %s is not a payload image\n", device); + return 1; + } + + if (read_exact(in, u32buf, 4) != 0) { + fclose(in); + fputs("payload-import: malformed payload header\n", stderr); + return 1; + } + file_count = read_u32le(u32buf); + if (file_count > 200000U) { + fclose(in); + fprintf(stderr, "payload-import: unreasonable file count: %u\n", file_count); + return 1; + } + + if (mkdir(dest_dir, 0755) != 0 && errno != EEXIST) { + fclose(in); + fprintf(stderr, "payload-import: cannot create %s: %s\n", dest_dir, strerror(errno)); + return 1; + } + + printf("payload-import: reading %u files from %s\n", file_count, device); + for (i = 0; i < file_count; ++i) { + unsigned int name_len; + unsigned int data_len; + char *name; + char out_path[4096]; + FILE *out; + + if (read_exact(in, u32buf, 4) != 0) { + fclose(in); + fputs("payload-import: truncated entry header\n", stderr); + return 1; + } + name_len = read_u32le(u32buf); + if (read_exact(in, u32buf, 4) != 0) { + fclose(in); + fputs("payload-import: truncated entry size\n", stderr); + return 1; + } + data_len = read_u32le(u32buf); + + if (name_len == 0 || name_len > MAX_NAME_LEN) { + fclose(in); + fprintf(stderr, "payload-import: invalid name length %u\n", name_len); + return 1; + } + + name = (char *)malloc(name_len + 1); + if (name == NULL) { + fclose(in); + fputs("payload-import: out of memory\n", stderr); + return 1; + } + + if (read_exact(in, name, name_len) != 0) { + free(name); + fclose(in); + fputs("payload-import: truncated file name\n", stderr); + return 1; + } + name[name_len] = 0; + + if (!is_valid_name(name)) { + fclose(in); + fprintf(stderr, "payload-import: invalid payload file name: %s\n", name); + free(name); + return 1; + } + + if (snprintf(out_path, sizeof(out_path), "%s/%s", dest_dir, name) >= (int)sizeof(out_path)) { + free(name); + fclose(in); + fputs("payload-import: output path too long\n", stderr); + return 1; + } + + out = fopen(out_path, "wb"); + if (out == NULL) { + fprintf(stderr, "payload-import: cannot write %s: %s\n", out_path, strerror(errno)); + free(name); + fclose(in); + return 1; + } + + if (copy_exact(in, out, data_len) != 0) { + fprintf(stderr, "payload-import: failed while copying %s\n", name); + free(name); + fclose(out); + fclose(in); + return 1; + } + + fclose(out); + printf("payload-import: %s\n", name); + free(name); + } + + fclose(in); + return 0; +} + +static int import_from_first_payload(const char *dest_dir) +{ + const char *prefixes[] = {"/dev/sd", "/dev/vd", "/dev/hd"}; + int p; + + for (p = 0; p < 3; ++p) { + char letter; + for (letter = 'b'; letter <= 'z'; ++letter) { + char device[16]; + if (snprintf(device, sizeof(device), "%s%c", prefixes[p], letter) >= (int)sizeof(device)) { + continue; + } + if (access(device, R_OK) != 0) { + continue; + } + if (has_payload_magic(device) == 0) { + return extract_payload(device, dest_dir); + } + } + } + + fputs("payload-import: no payload disk found\n", stderr); + return 2; +} + +static void usage(const char *name) +{ + fprintf(stderr, + "Usage:\n" + " %s --probe \n" + " %s [--device ] \n", + name, name); +} + +int main(int argc, char **argv) +{ + const char *device = NULL; + const char *dest_dir = NULL; + int i; + + if (argc == 3 && strcmp(argv[1], "--probe") == 0) { + return has_payload_magic(argv[2]); + } + + for (i = 1; i < argc; ++i) { + if (strcmp(argv[i], "--device") == 0) { + i += 1; + if (i >= argc) { + usage(argv[0]); + return 1; + } + device = argv[i]; + } else if (dest_dir == NULL) { + dest_dir = argv[i]; + } else { + usage(argv[0]); + return 1; + } + } + + if (dest_dir == NULL) { + usage(argv[0]); + return 1; + } + + if (device != NULL) { + return extract_payload(device, dest_dir); + } + return import_from_first_payload(dest_dir); +}