Add options for;

- disk to be created (blank disk given to live-bootstrap) (default)
- disk to already exist but sources downloaded within live-bootstrap
- sources to be downloaded outside live-bootstrap (non-blank disk given
  to live-bootstrap)

Also migrate sysb to use sys_transfer in QEMU mode also.

Note that this means copy_sysc is now irrelevant. sysc is *always*
sourced from sysa.

lib/utils.py

@@ -32,10 +32,11 @@ def create_disk(image, disk_type, fs_type, size):
     loop_dev = run('sudo', 'losetup', '-f', capture_output=True).stdout.decode().strip()
     run('sudo', 'losetup', '-P', loop_dev, image)
     # Create the partition
-    run('sudo', 'parted', '--script', image, 'mklabel', disk_type, 'mkpart',
+    if disk_type != "none":
-            'primary', 'ext4', '0%', '100%')
+        run('sudo', 'parted', '--script', image, 'mklabel', disk_type, 'mkpart',
-    run('sudo', 'partprobe', loop_dev)
+                'primary', 'ext4', '0%', '100%')
-    run('sudo', 'mkfs.' + fs_type, loop_dev + "p1")
+        run('sudo', 'partprobe', loop_dev)
+        run('sudo', 'mkfs.' + fs_type, loop_dev + "p1")
     return loop_dev
 
 def mount(source, target, fs_type, options='', **kwargs):

rootfs.py

@@ -33,7 +33,10 @@ def create_configuration_file(args):
         config.write("FORCE_TIMESTAMPS=" + str(args.force_timestamps) + "\n")
         config.write("CHROOT=" + str(args.chroot or args.bwrap) + "\n")
         config.write("UPDATE_CHECKSUMS=" + str(args.update_checksums) + "\n")
-        config.write("DISK=sda1\n")
+        if args.external_sources:
+            config.write("DISK=sda1\n")
+        else:
+            config.write("DISK=sda\n")
 
 def main():
     """
@@ -57,6 +60,8 @@ def main():
     parser.add_argument("--update-checksums",
                         help="Update checksum files.",
                         action="store_true")
+    parser.add_argument("--external-sources",
+                        help="Download sources externally from live-bootstrap.")
     parser.add_argument("--no-create-config",
                         help="Do not automatically create config file",
                         action="store_true")
@@ -116,7 +121,7 @@ def main():
             pass
 
     system_c = SysC(arch=args.arch, preserve_tmp=args.preserve,
-                    tmpdir=args.tmpdir)
+                    tmpdir=args.tmpdir, external_sources=args.external_sources)
     system_b = SysB(arch=args.arch, preserve_tmp=args.preserve)
     system_a = SysA(arch=args.arch, preserve_tmp=args.preserve,
                     tmpdir=args.tmpdir,
@@ -138,7 +143,6 @@ print(shutil.which('chroot'))
         system_c.prepare(mount_tmpfs=True,
                          create_disk_image=False)
         system_a.prepare(mount_tmpfs=True,
-                         copy_sysc=True,
                          create_initramfs=False,
                          repo_path=args.repo)
 
@@ -151,7 +155,6 @@ print(shutil.which('chroot'))
         system_c.prepare(mount_tmpfs=False,
                          create_disk_image=False)
         system_a.prepare(mount_tmpfs=False,
-                         copy_sysc=True,
                          create_initramfs=False,
                          repo_path=args.repo)
 
@@ -187,7 +190,6 @@ print(shutil.which('chroot'))
         system_c.prepare(mount_tmpfs=True,
                          create_disk_image=True)
         system_a.prepare(mount_tmpfs=True,
-                         copy_sysc=False,
                          create_initramfs=True,
                          repo_path=args.repo)
 
@@ -211,7 +213,6 @@ print(shutil.which('chroot'))
         system_c.prepare(mount_tmpfs=True,
                          create_disk_image=True)
         system_a.prepare(mount_tmpfs=True,
-                         copy_sysc=False,
                          create_initramfs=True,
                          repo_path=args.repo)
 
@@ -223,7 +224,6 @@ print(shutil.which('chroot'))
         system_c.prepare(mount_tmpfs=True,
                          create_disk_image=True)
         system_a.prepare(mount_tmpfs=True,
-                         copy_sysc=False,
                          create_initramfs=True,
                          repo_path=args.repo)
 
@@ -232,6 +232,7 @@ print(shutil.which('chroot'))
             '-m', str(args.qemu_ram) + 'M',
             '-no-reboot',
             '-hda', system_c.dev_name,
+            '-nic', 'user,ipv6=off,model=e1000',
             '-kernel', args.kernel,
             '-initrd', system_a.initramfs_path,
             '-nographic',

sysa.py

@@ -18,7 +18,7 @@ class SysA(SysGeneral):
     Class responsible for preparing sources for System A.
     """
     # pylint: disable=too-many-instance-attributes,too-many-arguments
-    def __init__(self, arch, preserve_tmp, tmpdir, sysb_dir, sysc_tmp):
+    def __init__(self, arch, preserve_tmp, tmpdir, sysb_dir, sysc_dir):
         self.git_dir = os.path.dirname(os.path.join(__file__))
         self.arch = arch
         self.preserve_tmp = preserve_tmp
@@ -32,9 +32,9 @@ class SysA(SysGeneral):
         self.base_dir = self.sysa_dir
         self.cache_dir = os.path.join(self.sys_dir, 'distfiles')
         self.sysb_dir = sysb_dir
-        self.sysc_tmp = sysc_tmp
+        self.sysc_dir = sysc_dir
 
-    def prepare(self, mount_tmpfs, copy_sysc, create_initramfs, repo_path):
+    def prepare(self, mount_tmpfs, create_initramfs, repo_path):
         """
         Prepare directory structure for System A.
         We create an empty tmp directory, unpack stage0-posix.
@@ -51,8 +51,7 @@ class SysA(SysGeneral):
         # sysb must be added to sysa as it is another initramfs stage
         self.sysb()
 
-        if copy_sysc:
+        self.sysc()
-            self.sysc()
 
         if repo_path:
             repo_dir = os.path.join(self.tmp_dir, 'usr', 'src', 'repo-preseeded')
@@ -66,17 +65,17 @@ class SysA(SysGeneral):
         self.get_packages()
 
         shutil.copytree(self.sys_dir, os.path.join(self.tmp_dir, 'sysa'),
-                shutil.ignore_patterns('tmp'))
+                ignore=shutil.ignore_patterns('tmp'))
 
     def sysb(self):
         """Copy in sysb files for sysb."""
         shutil.copytree(self.sysb_dir, os.path.join(self.tmp_dir, 'sysb'),
-                shutil.ignore_patterns('tmp'))
+                ignore=shutil.ignore_patterns('tmp'))
 
     def sysc(self):
         """Copy in sysc files for sysc."""
-        shutil.copytree(self.sysc_tmp, os.path.join(self.tmp_dir, 'sysc'),
+        shutil.copytree(self.sysc_dir, os.path.join(self.tmp_dir, 'sysc'),
-                shutil.ignore_patterns('tmp'))
+                ignore=shutil.ignore_patterns('tmp', 'distfiles'))
 
     def stage0_posix(self):
         """Copy in all of the stage0-posix"""

sysa/helpers.sh

@@ -412,6 +412,7 @@ populate_device_nodes() {
 
 sys_transfer() {
     local dest=$1
+    local sys_sources=$2
 
     mkdir -p "${dest}/${PREFIX}/bin" "${dest}/${PREFIX}/src"
 
@@ -419,11 +420,13 @@ sys_transfer() {
     cp "${PREFIX}/bin/bash" "${PREFIX}/bin/tar" "${PREFIX}/bin/bzip2" "${dest}${PREFIX}/bin/"
 
     # Transfer misc files
-    cp "${SOURCES}/helpers.sh" "${SOURCES}/SHA256SUMS.pkgs" "${SOURCES}/bootstrap.cfg" "${dest}/"
+    cp "${SOURCES}/helpers.sh" "${SOURCES}/SHA256SUMS.pkgs" "${SOURCES}/bootstrap.cfg" "${dest}/${PREFIX}/src"
 
-    cp -r "${PREFIX}/src/" "${dest}${PREFIX}/"
+    cp -r "${sys_sources}/*" "${dest}/${PREFIX}/src"
+    cp -f "${sys_sources}/init" "${dest}/"
+    cp -r "${PREFIX}/src/repo" "${dest}/${PREFIX}/src"
 
-    shift
+    shift 2
     # Copy additional binaries
     set -- "${@/#/${PREFIX}/bin/}"
     cp "$@" "${dest}${PREFIX}/bin/"

sysa/run.sh

@@ -22,16 +22,11 @@ export SRCDIR="${srcdir}"
 create_sysb() {
     # Copy everything in
     echo "Creating sysb rootfs"
-    mkdir -p "/sysb${PREFIX}"
+    sys_transfer /sysb_image /sysb
-    for d in bin include lib libexec share src; do
+    cp -rl /sysc /sysb/sysc_src
-        # Minimise RAM (storage) use - use hard links
-        cp -rl "${PREFIX}/${d}" "/sysb${PREFIX}/${d}"
-    done
-    cp "${SOURCES}/helpers.sh" "${SOURCES}/SHA256SUMS.pkgs" "${SOURCES}/bootstrap.cfg" "/sysb/${SRCDIR}"
-    populate_device_nodes /sysb
     echo "Creating sysb initramfs"
     gen_initramfs_list.sh -o "${PREFIX}/boot/initramfs-sysb.cpio.gz" /sysb
-    rm -rf /sysb # Cleanup
+    rm -rf /sysb /sysb_image # Cleanup
 }
 
 go_sysb() {
@@ -256,6 +251,6 @@ if [ "${CHROOT}" = False ]; then
 fi
 
 # In chroot mode transition directly into System C.
-SYSC=/sysc
+SYSC=/sysc_image
-sys_transfer "${SYSC}" gzip patch
+sys_transfer "${SYSC}" /sysc gzip patch
 exec chroot "${SYSC}" /init

sysb/init

@@ -7,7 +7,31 @@
 
 set -e
 
+. /usr/src/helpers.sh
+
+export PATH=/usr/bin
+export PREFIX=/usr
+export SOURCES=/usr/src
+
+echo 
+echo "Installing packages into sysb"
+
+install_tar() {
+    echo "${1}: installing package"
+    src_apply_tar "$@"
+}
+
+# Install needed packages.
+install_tar coreutils-5.0 0
+install_tar sed-4.0.9 0
+
+install_tar bzip2-1.0.8 0
+install_tar coreutils-6.10 0
+install_tar e2fsprogs-1.45.7 0
+install_tar grep-2.4 0
+install_tar kexec-tools-2.0.22 0
+install_tar util-linux-2.19.1 0
+
 # Begin sysb bootstrapping process
-mv ./run.sh /usr/src
 cd /usr/src
 ./run.sh

sysb/run.sh

@@ -34,10 +34,15 @@ create_hdx
 ask_disk() {
     echo
     echo "What disk would you like to use for live-bootstrap?"
-    echo "(live-bootstrap assumes you have pre-prepared the disk)."
+    echo "This disk may have pre-prepared sources on it."
-    echo "Please provide in format sdxx (as you would find under /dev)."
+    echo "If there is no partition we will make one".
+    echo "Please provide in format sdxx (as you would find under /dev),"
+    echo "or sdx if it is a blank disk. An ext4 partition is expected on"
+    echo "existing disks."
     echo "You can type 'list' to get a list of disks to help you figure"
     echo "out which is the right disk."
+    echo "NO WARRANTY IS PROVIDED FOR BUGGY BEHAVIOUR, INCLUDING THAT"
+    echo "REGARDING DISKS & DATA."
     echo
     read -r DISK
 
@@ -57,6 +62,15 @@ if [ -z "${DISK}" ] || ! [ -e "/dev/${DISK}" ]; then
     echo "DISK=${DISK}" >> /usr/src/bootstrap.cfg
 fi
 
+if [ -z "$(fdisk -l "/dev/${DISK}" | grep -E "${DISK}p?[0-9]")" ]; then
+    echo "Creating partition table and partition"
+    echo ";" | sfdisk "/dev/${DISK}"
+    mkfs.ext4 "/dev/${DISK}1"
+    DISK="${DISK}1"
+fi
+
+echo "export DISK=${DISK}" >> /usr/src/bootstrap.cfg
+
 PREFIX=/usr
 SOURCES="${PREFIX}/src"
 SYSC=/sysc
@@ -68,7 +82,7 @@ mount -t ext4 "/dev/${DISK}" /sysc
 
 # Copy over appropriate data
 echo "Copying data into sysc"
-sys_transfer "${SYSC}" gzip patch
+sys_transfer "${SYSC}" /sysc_src gzip patch
 sync
 
 # switch_root into sysc 1. for simplicity 2. to avoid kexecing again

sysc.py

@@ -21,10 +21,11 @@ class SysC(SysGeneral):
     dev_name = None
 
     # pylint: disable=too-many-instance-attributes
-    def __init__(self, arch, preserve_tmp, tmpdir):
+    def __init__(self, arch, preserve_tmp, tmpdir, external_sources):
         self.git_dir = os.path.dirname(os.path.join(__file__))
         self.arch = arch
         self.preserve_tmp = preserve_tmp
+        self.external_sources = external_sources
 
         self.sys_dir = os.path.join(self.git_dir, 'sysc')
         self.cache_dir = os.path.join(self.sys_dir, 'distfiles')
@@ -55,18 +56,154 @@ class SysC(SysGeneral):
         if create_disk_image:
             # Create + mount a disk for QEMU to use
             disk_path = os.path.join(self.tmp_dir, 'disk.img')
-            self.dev_name = create_disk(disk_path, "msdos", "ext4", '8G')
+            if self.external_sources:
-            rootfs_dir = os.path.join(self.tmp_dir, 'mnt')
+                self.dev_name = create_disk(disk_path, "msdos", "ext4", '8G')
-            os.mkdir(rootfs_dir)
+                self.rootfs_dir = os.path.join(self.tmp_dir, 'mnt')
-            mount(self.dev_name + "p1", rootfs_dir, 'ext4')
+                os.mkdir(rootfs_dir)
+                mount(self.dev_name + "p1", self.rootfs_dir, 'ext4')
+            else:
+                self.dev_name = create_disk(disk_path, "none", "ext4", '8G')
             # Use chown to allow executing user to access it
             run('sudo', 'chown', getpass.getuser(), self.dev_name)
-            run('sudo', 'chown', getpass.getuser(), rootfs_dir)
+            if self.external_sources:
+                run('sudo', 'chown', getpass.getuser(), rootfs_dir)
         else:
             rootfs_dir = self.tmp_dir
 
-        copytree(self.sys_dir, rootfs_dir, ignore=shutil.ignore_patterns("tmp"))
+        if self.external_sources:
+            self.get_packages()
 
         # Unmount tmp/mnt if it was mounted
         if create_disk_image:
             umount(rootfs_dir)
+
+    # pylint: disable=line-too-long,too-many-statements
+    def get_packages(self):
+        """Prepare remaining sources"""
+        # bash 5.1
+        self.get_file("https://mirrors.kernel.org/gnu/bash/bash-5.1.tar.gz")
+
+        # xz 5.0.5
+        self.get_file("https://ixpeering.dl.sourceforge.net/project/lzmautils/xz-5.0.5.tar.bz2")
+
+        # automake 1.11.2
+        self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.11.2.tar.bz2")
+
+        # autoconf 2.69
+        self.get_file("https://mirrors.kernel.org/gnu/autoconf/autoconf-2.69.tar.xz")
+
+        # automake 1.15.1
+        self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.15.1.tar.xz")
+
+        # tar 1.34
+        self.get_file(["https://mirrors.kernel.org/gnu/tar/tar-1.34.tar.xz",
+                       "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-30820c.tar.gz"])
+
+        # coreutils 8.32
+        self.get_file(["https://git.savannah.gnu.org/cgit/coreutils.git/snapshot/coreutils-8.32.tar.gz",
+                       "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-d279bc.tar.gz"])
+
+        # pkg-config 0.29.2
+        self.get_file("http://gentoo.osuosl.org/distfiles/pkg-config-0.29.2.tar.gz")
+
+        # make 4.2.1
+        self.get_file("https://ftp.gnu.org/gnu/make/make-4.2.1.tar.gz")
+
+        # gmp 6.2.1
+        self.get_file("https://mirrors.kernel.org/gnu/gmp/gmp-6.2.1.tar.xz")
+
+        # autoconf archive 2021.02.19
+        self.get_file("https://mirrors.kernel.org/gnu/autoconf-archive/autoconf-archive-2021.02.19.tar.xz")
+
+        # mpfr 4.1.0
+        self.get_file("https://mirrors.kernel.org/gnu/mpfr/mpfr-4.1.0.tar.xz")
+
+        # mpc 1.2.1
+        self.get_file("https://mirrors.kernel.org/gnu/mpc/mpc-1.2.1.tar.gz")
+
+        # flex 2.5.33
+        self.get_file("http://download.nust.na/pub2/openpkg1/sources/DST/flex/flex-2.5.33.tar.gz")
+
+        # bison 2.3
+        self.get_file(["https://mirrors.kernel.org/gnu/bison/bison-2.3.tar.bz2",
+                       "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-b28236b.tar.gz"])
+
+        # bison 3.4.2
+        self.get_file(["https://mirrors.kernel.org/gnu/bison/bison-3.4.2.tar.xz",
+                       "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-672663a.tar.gz"])
+
+        # perl 5.10.5
+        self.get_file("https://www.cpan.org/src/5.0/perl-5.10.1.tar.bz2")
+
+        # dist 3.5-236
+        # Debian's version is used because upstream is not to be found (dead?)
+        self.get_file("http://deb.debian.org/debian/pool/main/d/dist/dist_3.5-236.orig.tar.gz",
+                output="dist-3.5-236.tar.gz")
+
+        # perl 5.32.1
+        self.get_file(["https://www.cpan.org/src/5.0/perl-5.32.1.tar.xz",
+                       "http://deb.debian.org/debian/pool/main/p/perl/perl_5.32.1.orig-regen-configure.tar.gz"])
+
+        # libarchive-3.5.2
+        self.get_file("https://libarchive.org/downloads/libarchive-3.5.2.tar.xz")
+
+        # openssl-1.1.1l
+        self.get_file("https://artfiles.org/openssl.org/source/old/1.1.1/openssl-1.1.1l.tar.gz")
+
+        # curl 7.83.0
+        self.get_file("https://master.dl.sourceforge.net/project/curl.mirror/curl-7_83_0/curl-7.83.0.tar.xz?viasf=1",
+                output="curl-7.83.0.tar.xz")
+
+        # ca-certificates-3.78
+        self.get_file("https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_78_RTM/src/nss-3.78.tar.gz",
+                output="ca-certificates-3.78.tar.gz")
+
+        # xbps 0.59.1
+        self.get_file("https://github.com/void-linux/xbps/archive/refs/tags/0.59.1.tar.gz",
+                       output="xbps-0.59.1.tar.gz")
+
+        # autoconf 2.71
+        self.get_file("https://mirrors.kernel.org/gnu/autoconf/autoconf-2.71.tar.xz")
+
+        # automake 1.16.3
+        self.get_file("https://mirrors.kernel.org/gnu/automake/automake-1.16.3.tar.xz")
+
+        # patch 2.7.6
+        self.get_file(["https://mirrors.kernel.org/gnu/patch/patch-2.7.6.tar.xz",
+                       "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-e017871.tar.gz"])
+
+        # gettext 0.21
+        self.get_file(["https://mirrors.kernel.org/gnu/gettext/gettext-0.21.tar.xz",
+                       "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-7daa86f.tar.gz"])
+
+        # texinfo 6.7
+        self.get_file(["https://mirrors.kernel.org/gnu/texinfo/texinfo-6.7.tar.xz",
+                       "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-b81ec69.tar.gz"])
+
+        # zlib 1.2.12
+        self.get_file("https://www.zlib.net/zlib-1.2.12.tar.xz")
+
+        # gcc 4.7.4
+        self.get_file("https://mirrors.kernel.org/gnu/gcc/gcc-4.7.4/gcc-4.7.4.tar.bz2")
+
+        # gperf 3.1
+        self.get_file("https://mirrors.kernel.org/gnu/gperf/gperf-3.1.tar.gz")
+
+        # libunistring 0.9.10
+        self.get_file(["https://mirrors.kernel.org/gnu/libunistring/libunistring-0.9.10.tar.xz",
+                       "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-52a06cb3.tar.gz"])
+
+        # libffi 3.3
+        self.get_file("https://github.com/libffi/libffi/releases/download/v3.3/libffi-3.3.tar.gz")
+
+        # libatomic_ops 7.6.10
+        self.get_file("https://github.com/ivmai/libatomic_ops/releases/download/v7.6.10/libatomic_ops-7.6.10.tar.gz")
+
+        # boehm-gc 8.0.4
+        self.get_file("https://www.hboehm.info/gc/gc_source/gc-8.0.4.tar.gz")
+
+        # guile 3.0.7
+        self.get_file(["https://mirrors.kernel.org/gnu/guile/guile-3.0.7.tar.xz",
+                       "https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-901694b9.tar.gz",
+                       "https://github.com/schierlm/guile-psyntax-bootstrapping/archive/refs/tags/guile-3.0.7.tar.gz"],
+                       output=["guile-3.0.7.tar.xz", "gnulib-901694b9.tar.gz", "guile-psyntax-bootstrapping.tar.gz"])

sysc/init

@@ -7,7 +7,7 @@
 set -e
 
 # shellcheck source=sysa/helpers.sh
-. helpers.sh
+. /usr/src/helpers.sh
 
 export PATH=/usr/bin:/usr/sbin
 export PREFIX=/usr
@@ -54,12 +54,6 @@ install_tar util-linux-2.19.1 0
 # Fix invocation of bash from perl
 ln -s /usr/bin/bash /usr/bin/sh
 
-# Prepare sysc directory structure
-mkdir -p /usr/src
-mv run*.sh /usr/src
-mv helpers.sh SHA256SUMS.pkgs bootstrap.cfg /usr/src
-mv /*-* /usr/src # build scripts
-
 # Check tarballs
 cd "${DISTFILES}"
 sha256sum -c /SHA256SUMS.sources