Merge pull request #493 from fosslinux/mirrors

Mirrors
2026-03-02 01:18:08 +01:00 · 2025-02-03 10:13:46 +11:00 · 2025-02-03 10:13:46 +11:00 · b934782b5f
commit b934782b5f
parent 3896c1dfc9 ef5477c74a
62 changed files with 582 additions and 340 deletions
--- a/.github/workflows/bwrap.yml
+++ b/.github/workflows/bwrap.yml
@ -32,20 +32,24 @@ jobs:
        uses: actions/cache/restore@v4
        with:
          path: |
-            distfiles
+            mirror
+            mirror-state
          key: cache-${{ hashFiles('steps/*/sources') }}
+          restore-keys: |
+            cache-
      - name: Get sources
        if: steps.cache.outputs.cache-hit != 'true'
-        run: ./download-distfiles.sh
+        run: mkdir -p mirror mirror-state && ./mirror.sh mirror mirror-state
      - name: Cache sources
        if: steps.cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: |
-            distfiles
+            mirror
+            mirror-state
          key: cache-${{ hashFiles('steps/*/sources') }}
      - name: Run bootstrap
-        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1
+        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass1 --mirror file://${PWD}/mirror
      - name: Archive created packages
        if: failure() # archive failed builds progress
        uses: actions/upload-artifact@v4
@ -87,18 +91,12 @@ jobs:
        uses: actions/cache/restore@v4
        with:
          path: |
-            distfiles
-          key: cache-${{ hashFiles('steps/*/sources') }}
-      - name: Get sources
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: ./download-distfiles.sh
-      - name: Cache sources
-        if: steps.cache.outputs.cache-hit != 'true'
-        uses: actions/cache/save@v4
-        with:
-          path: |
-            distfiles
+            mirror
+            mirror-state
          key: cache-${{ hashFiles('steps/*/sources') }}
+          fail-on-cache-miss: true
+      - name: Copy distfiles
+        run: ./download-distfiles.sh file:///${PWD}/mirror
      - name: Run bootstrap
        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass2
      - name: Archive created packages
@ -142,20 +140,14 @@ jobs:
        uses: actions/cache/restore@v4
        with:
          path: |
-            distfiles
-          key: cache-${{ hashFiles('steps/*/sources') }}
-      - name: Get sources
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: ./download-distfiles.sh
-      - name: Cache sources
-        if: steps.cache.outputs.cache-hit != 'true'
-        uses: actions/cache/save@v4
-        with:
-          path: |
-            distfiles
+            mirror
+            mirror-state
          key: cache-${{ hashFiles('steps/*/sources') }}
+          fail-on-cache-miss: true
+      - name: Copy distfiles
+        run: ./download-distfiles.sh file:///${PWD}/mirror
      - name: Run bootstrap
-        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3
+        run: ./rootfs.py --bwrap --external-sources --build-kernels --cores 2 --internal-ci pass3 --mirror file://${PWD}/mirror
      - name: Archive created packages
        if: always() # archive both failed and successful builds
        uses: actions/upload-artifact@v4
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@ -21,7 +21,7 @@ jobs:
      - name: Install pylint
        run: sudo pip3 install pylint
      - name: pylint
-        run: pylint rootfs.py lib/utils.py lib/generator.py lib/target.py check_substitutes.py --disable=duplicate-code
+        run: pylint rootfs.py source_manifest.py lib/*.py --disable=duplicate-code

  shellcheck:
    name: Lint shell files
@ -30,7 +30,7 @@ jobs:
      - name: Checkout repo
        uses: actions/checkout@v3
      - name: shellcheck
-        run: shellcheck steps/helpers.sh download-distfiles.sh
+        run: shellcheck steps/helpers.sh download-distfiles.sh mirror.sh

  reuse:
    name: Lint reuse information
@ -42,12 +42,3 @@ jobs:
        run: sudo pip3 install reuse
      - name: reuse
        run: reuse lint
-
-  substitutes:
-    name: Check validity of substituted tarballs
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-      - name: Check validity of substitutes
-        run: ./check_substitutes.py
--- a/DEVEL.md
+++ b/DEVEL.md
@ -84,3 +84,43 @@ libraries that are compiled and installed.
  - Patches are licensed under the license of the project which they are
    patching.
  - All files (excluding files within submodules) must comply with REUSE v3.0.
+
+## `sources` file
+
+The format of the sources file for a HTTP source file is:
+
+```
+<url> <checksum> [filename]
+```
+
+For a Git repository snapshot:
+
+```
+<git url> <URL to HTTP snapshot> <checksum> [filename]
+```
+
+The format of a git url must always be `git://<path to repository>~<reference>`.
+`reference` is, for instance, a commit, tag or branch. Always use
+Git protocol path, except for GitHub, in which `git://` is automagically
+changed to `https://`.
+
+The URL to HTTP snapshot may be `_` (a single underscore) if no HTTP snapshot
+exists. In this case, the filename is compulsory. The checksum is of the Git
+snapshot, generated with `git archive`. See `mirror.sh` for more detailed rules.
+
+Some specific helpful things:
+- prefer `.tar.gz`
+- For GitHub, all snapshots are generated with long commit IDs, so use long
+  commit IDs.
+
+## Running a mirror
+
+There are 3 things you need to run a mirror;
+
+1. A tool to perform the mirroring (e.g. `mirror.sh` in this repository); we
+   encourage diversity in this area.
+2. A server on which to host the mirror; this should have HTTP support, and
+   optionally HTTPS. It is of greatly diminished utility if it does not support
+   plain HTTP.
+3. Automatic updates. This should involve updating the Git repository and
+   updating the mirror as required.
--- a/LICENSES/BSD-3-Clause-flex.txt
+++ b/LICENSES/BSD-3-Clause-flex.txt
@ -0,0 +1,42 @@
+Flex carries the copyright used for BSD software, slightly modified
+because it originated at the Lawrence Berkeley (not Livermore!) Laboratory,
+which operates under a contract with the Department of Energy:
+
+Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007 The Flex Project.
+
+Copyright (c) 1990, 1997 The Regents of the University of California.
+All rights reserved.
+
+This code is derived from software contributed to Berkeley by
+Vern Paxson.
+
+The United States Government has rights in this work pursuant
+to contract no. DE-AC03-76SF00098 between the United States
+Department of Energy and the University of California.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+Neither the name of the University nor the names of its contributors
+may be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.
+
+This basically says "do whatever you please with this software except
+remove this notice or take advantage of the University's (or the flex
+authors') name".
+
+Note that the "flex.skl" scanner skeleton carries no copyright notice.
+You are free to do whatever you please with scanners generated using flex;
+for them, you are not even bound by the above copyright.
--- a/check_substitutes.py
+++ b/check_substitutes.py
@ -1,162 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: GPL-3.0-or-later
-#
-# SPDX-FileCopyrightText: 2024 fosslinux <fosslinux@aussies.space>
-
-"""Check that substituted files are the same."""
-import bz2
-import filecmp
-import gzip
-import itertools
-import lzma
-import shutil
-import tarfile
-import tempfile
-import sys
-import os
-
-from lib.generator import Generator
-
-# Get a temporary directory to work in
-working = tempfile.mkdtemp()
-
-# Colour constants
-# pylint: disable=too-few-public-methods
-class Colors():
-    """ANSI Color Codes"""
-    GREY = "\033[90m"
-    RED = "\033[91m"
-    GREEN = "\033[92m"
-    ORANGE = "\033[91m\033[93m"
-    YELLOW = "\033[93m"
-    END = "\033[0m"
-
-def traverse_path(base_root):
-    """Takes a path and returns a set of all directories and files in that path."""
-    all_dirs = set()
-    all_files = set()
-    for root, directories, files in os.walk(base_root, topdown=True):
-        for d in directories:
-            all_dirs.add(os.path.join(root, d).lstrip(base_root))
-        for f in files:
-            all_files.add(os.path.join(root, f).lstrip(base_root))
-    return (all_dirs, all_files)
-
-class Distfile():
-    """Represents one distfile and operations performed on it."""
-    def __init__(self, i, url):
-        self.i = i
-        self.url = url
-        self.out_file = f"{i}-{os.path.basename(url)}"
-        self.filepath = ""
-
-    def download(self):
-        """Downloads the distfile."""
-        Generator.download_file(self.url, working, self.out_file, silent=True)
-        self.filepath = os.path.join(working, self.out_file)
-
-    def decompress(self):
-        """Decompresses the distfile."""
-        compression = self.out_file.rsplit('.', maxsplit=1)[-1]
-        decompress_func = {
-            "gz": gzip.open,
-            "tgz": gzip.open,
-            "bz2": bz2.open,
-            "xz": lzma.open,
-            "lzma": lzma.open
-        }
-        if compression not in decompress_func:
-            # No decompression needed
-            return
-        # Remove the compression extension
-        new_path = '.'.join(self.filepath.split('.')[:-1])
-        # tgz -> .tar
-        if compression == "tgz":
-            new_path = f"{new_path}.tar"
-        # Move the decompressed binary stream to a new file
-        with decompress_func[compression](self.filepath, 'rb') as fin:
-            with open(new_path, 'wb') as fout:
-                shutil.copyfileobj(fin, fout)
-        self.filepath = new_path
-
-    def extract(self):
-        """Extracts the distfile."""
-        # Sanity check
-        if not tarfile.is_tarfile(self.filepath):
-            return
-        out_dir = os.path.join(working, f"{self.i}")
-        os.mkdir(out_dir)
-        with tarfile.open(self.filepath, 'r') as f:
-            f.extractall(path=out_dir)
-        self.filepath = out_dir
-
-    # It makes more sense here to label them d1 and d2 rather than have one be self.
-    # pylint: disable=no-self-argument
-    def compare(d1, d2):
-        """Compares the distfile to another distfile."""
-        if not os.path.isdir(d1.filepath):
-            # Compare files
-            return filecmp.cmp(d1.filepath, d2.filepath, shallow=False)
-        if not os.path.isdir(d2.filepath):
-            # Then, d2 is a file and d1 is a directory
-            return False
-        # Otherwise it's two directories
-        dirnames1, filenames1 = traverse_path(d1.filepath)
-        dirnames2, filenames2 = traverse_path(d2.filepath)
-        if dirnames1 != dirnames2:
-            return False
-        if filenames1 != filenames2:
-            return False
-        return filecmp.cmpfiles(d1.filepath, d2.filepath, filenames1, shallow=False)
-
-def check(*args):
-    """Check if a list of distfiles are equivalent."""
-    notequiv = []
-    # Find all pairs that are not equivalent
-    for pair in itertools.combinations(args, 2):
-        if pair[0].compare(pair[1]):
-            print(f"{Colors.GREY}DEBUG: {pair[0].url} is equivalent to {pair[1].url}{Colors.END}")
-        else:
-            notequiv.append(pair)
-
-    # Decompress all, and check again
-    for d in {y for x in notequiv for y in x}:
-        d.decompress()
-    for pair in notequiv.copy():
-        if pair[0].compare(pair[1]):
-            # pylint: disable=line-too-long
-            print(f"{Colors.YELLOW}NOTE: {pair[0].url} is equivalent to {pair[1].url} when decompressed{Colors.END}")
-            notequiv.remove(pair)
-
-    # Extract all, and check again
-    for d in {y for x in notequiv for y in x}:
-        d.extract()
-    has_error = False
-    for pair in notequiv:
-        if pair[0].compare(pair[1]):
-            # pylint: disable=line-too-long
-            print(f"{Colors.ORANGE}WARN: {pair[0].url} is equivalent to {pair[1].url} when extracted{Colors.END}")
-        else:
-            has_error = True
-            # pylint: disable=line-too-long
-            print(f"{Colors.RED}ERROR: {pair[0].url} is not equivalent to {pair[1].url}!{Colors.END}")
-
-    return has_error
-
-def main():
-    """Main function."""
-    has_error = False
-    with open("substitutes", 'r', encoding="utf-8") as f:
-        for line in f.readlines():
-            urls = line.strip().split(' ')
-            distfiles = []
-            for i, url in enumerate(urls):
-                distfiles.append(Distfile(i, url))
-            for distfile in distfiles:
-                distfile.download()
-            if check(*distfiles):
-                has_error = True
-    sys.exit(has_error)
-
-if __name__ == "__main__":
-    main()
--- a/download-distfiles.sh
+++ b/download-distfiles.sh
@ -4,35 +4,58 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later

-download_source() {
-    local distfiles="${1}"
-    local url="${2}"
-    local checksum="${3}"
-    local fname="${4}"
-    # Default to basename of url if not given
-    fname="${fname:-$(basename "${url}")}"
+# Optional arguments: Mirrors

-    local dest_path="${distfiles}/${fname}"
+download_source() {
+    local distfiles=${1}
+    local url=${2}
+    shift 2
+    if [[ ${url} == git://* ]]; then
+        url=${1}
+        shift
+    fi
+    local checksum=${1}
+    local fname=${2}
+    # Default to basename of url if not given
+    fname=${fname:-$(basename "${url}")}
+    if [ "${fname}" = "_" ]; then
+        echo "ERROR: ${url} must have a filename specified"
+        exit 1
+    fi
+
+    local dest_path=${distfiles}/${fname}
    if ! [ -e "${dest_path}" ]; then
        echo "Downloading ${fname}"
+        if [ "${mirrors_len}" -ne 0 ]; then
+            local mirror_ix=$((RANDOM % mirrors_len))
+            url=${mirrors[${mirror_ix}]}/${fname}
+        fi
        curl --fail --location "${url}" --output "${dest_path}" || true
    fi
 }

 check_source() {
-    local distfiles="${1}"
-    local url="${2}"
-    local checksum="${3}"
-    local fname="${4}"
+    local distfiles=${1}
+    local url=${2}
+    shift 2
+    if [[ ${url} == git://* ]]; then
+        url=${1}
+        shift
+    fi
+    local checksum=${1}
+    local fname=${2}
    # Default to basename of url if not given
-    fname="${fname:-$(basename "${url}")}"
+    fname=${fname:-$(basename "${url}")}

-    local dest_path="${distfiles}/${fname}"
+    local dest_path=${distfiles}/${fname}
    echo "${checksum}  ${dest_path}" | sha256sum -c
 }

 set -e

+mirrors=( "$@" )
+mirrors_len=$#
+
 cd "$(dirname "$(readlink -f "$0")")"
 mkdir -p distfiles

--- a/lib/generator.py
+++ b/lib/generator.py
@ -10,9 +10,11 @@ This file contains all code required to generate the boot image for live-bootstr

 import hashlib
 import os
+import random
 import shutil
 import tarfile
 import traceback
+
 import requests

 # pylint: disable=too-many-instance-attributes
@ -24,11 +26,13 @@ class Generator():
    git_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..')
    distfiles_dir = os.path.join(git_dir, 'distfiles')

-    def __init__(self, arch, external_sources, early_preseed, repo_path):
+    # pylint: disable=too-many-arguments,too-many-positional-arguments
+    def __init__(self, arch, external_sources, early_preseed, repo_path, mirrors):
        self.arch = arch
        self.early_preseed = early_preseed
        self.external_sources = external_sources
        self.repo_path = repo_path
+        self.mirrors = mirrors
        self.source_manifest = self.get_source_manifest(not self.external_sources)
        self.early_source_manifest = self.get_source_manifest(True)
        self.target_dir = None
@ -278,8 +282,7 @@ actual:   {readable_hash}\n\
 When in doubt, try deleting the file in question -- it will be downloaded again when running \
 this script the next time")

-    @staticmethod
-    def download_file(url, directory, file_name, silent=False):
+    def download_file(self, url, directory, file_name, silent=False):
        """
        Download a single source archive.
        """
@ -297,14 +300,33 @@ this script the next time")
        if not os.path.isfile(abs_file_name):
            if not silent:
                print(f"Downloading: {file_name}")
-            response = requests.get(url, allow_redirects=True, stream=True,
-                    headers=headers, timeout=20)
-            if response.status_code == 200:
-                with open(abs_file_name, 'wb') as target_file:
-                    target_file.write(response.raw.read())
+
+            def do_download(source):
+                response = requests.get(source, allow_redirects=True, stream=True,
+                        headers=headers, timeout=20)
+                if response.status_code == 200:
+                    with open(abs_file_name, 'wb') as target_file:
+                        target_file.write(response.raw.read())
+                    return True
+                print(f"Download failed from {option}: {response.status_code} {response.reason}")
+                return False
+
+            done = False
+            if self.mirrors:
+                options = [f"{x}/{file_name}" for x in self.mirrors]
            else:
-                raise requests.HTTPError("Download failed: HTTP " +
-                    str(response.status_code) + " " + response.reason)
+                options = []
+            random.shuffle(options)
+            for option in options:
+                if do_download(option):
+                    done = True
+                    break
+
+            if not done:
+                if url == "_" or not do_download(url):
+                    raise requests.RequestException(f"Unable to download {url} from ",
+                                                    "any mirror or original")
+
        return abs_file_name

    def get_packages(self):
@ -344,6 +366,9 @@ this script the next time")
                        for source in sources.readlines():
                            source = source.strip().split(" ")

+                            if source[0].startswith("git://"):
+                                source = source[1:]
+
                            if len(source) > 2:
                                file_name = source[2]
                            else:
--- a/lib/simple_mirror.py
+++ b/lib/simple_mirror.py
@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+"""
+This creates a simple "mirror" from a directory
+"""
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2025 fosslinux <fosslinux@aussies.space>
+
+import http.server
+import socketserver
+
+class SimpleMirror(socketserver.TCPServer):
+    """Simple HTTP mirror from a directory"""
+    def __init__(self, directory: str):
+        self.directory = directory
+        super().__init__(("localhost", 0), self._handler)
+
+    @property
+    def port(self):
+        """Port the HTTP server of the mirror is running on"""
+        return self.server_address[1]
+
+    def _handler(self, *args, **kwargs):
+        return http.server.SimpleHTTPRequestHandler(*args, directory=self.directory, **kwargs)
--- a/lib/utils.py
+++ b/lib/utils.py
@ -12,7 +12,7 @@ import shutil
 import subprocess
 import sys

-def run(*args, **kwargs):
+def run(*args, cleanup=None, **kwargs):
    """A small wrapper around subprocess.run"""
    arguments = [str(arg) for arg in args if arg is not None]

@ -23,6 +23,8 @@ def run(*args, **kwargs):
        return subprocess.run(arguments, check=True, **kwargs)
    except subprocess.CalledProcessError:
        print("Bootstrapping failed")
+        if cleanup:
+            cleanup()
        sys.exit(1)

 def run_as_root(*args, **kwargs):
--- a/mirror.sh
+++ b/mirror.sh
@ -0,0 +1,186 @@
+#!/bin/sh
+# This script is intentionally written in POSIX sh to be cross-platform.
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2025 fosslinux <fosslinux@aussies.space>
+
+set -e
+
+# Check arguments, etc
+if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then
+    echo "Usage: $0 <destination> [state]"
+    exit 1
+fi
+
+dest=$1
+if [ ! -d "${dest}" ]; then
+    echo "${dest} must be a directory"
+    exit 1
+fi
+
+if [ ! -w "${dest}" ]; then
+    echo "you must be able to write to ${dest}"
+    exit 1
+fi
+dest=$(realpath "${dest}")
+
+state=$2
+if [ "${state}" = "" ]; then
+    state="${PWD}/mirrorstate"
+fi
+state="$(realpath "${state}")"
+
+#######
+
+# Download a HTTP file
+download_file() {
+    url=${1}
+    out=${2}
+    # Download the file
+    continue_arg=""
+    if [ -e "${out}" ]; then
+        continue_arg="-C -"
+    fi
+    # shellcheck disable=SC2086
+    curl -L ${continue_arg} -o "${out}" "${url}"
+}
+
+# Check if a git reference exists in a given repository
+git_ref_exists() {
+    repo=${1}
+    ref=${2}
+    # change this to git show-ref once it is sufficiently not-new
+    ( cd "${repo}" || exit && git cat-file -t "${ref}" >/dev/null 2>&1 )
+    return $?
+}
+
+checksum_file() {
+    sha256sum "${1}" | cut -d' ' -f1
+}
+
+do_file() {
+    uri=${1}
+    echo "${uri}"
+
+    if echo "${uri}" | grep -qE "^https?://"; then
+        # HTTP file
+        checksum=${2}
+        filename=${3}
+        if [ "${filename}" = "" ]; then
+            filename=$(basename "${uri}")
+        fi
+
+        # Check if the file is already downloaded & the checksum is the same
+        dest_file=${dest}/${filename}
+        if [ -e "${dest_file}" ]; then
+            existing_checksum=$(checksum_file "${dest_file}")
+            if [ "${checksum}" = "${existing_checksum}" ]; then
+                # There is nothing we need to do here
+                return
+            fi
+        fi
+
+        # Attempt up to 3 times
+        retries=3
+        matching=no
+        while [ "${retries}" -gt 0 ]; do
+            download_file "${uri}" "${dest}/${filename}"
+            my_checksum=$(checksum_file "${dest_file}")
+            if [ "${checksum}" = "${my_checksum}" ]; then
+                matching="yes"
+                break
+            fi
+            retries=$((retries - 1))
+            if [ "${retries}" -gt 0 ]; then
+                echo "${uri}: checksum did not match, trying again"
+                rm "${dest}/${filename}"
+            fi
+            sleep 1
+        done
+
+        if [ "${matching}" = "no" ]; then
+            echo "${uri}: checksums do not match"
+            exit 1
+        fi
+    elif echo "${uri}" | grep -qE "^git://"; then
+        # Creating a tarball from a git repository
+
+        # Very unfortunately, different sites have different rules.
+        uri_path=${uri#git://}
+        # GitHub does not have git:// protocol support
+        if echo "${uri}" | grep -Eq "^git://github.com"; then
+            uri=https://${uri_path}
+        fi
+        repo=${uri%~*}
+        outdir=${state}/git/${repo#*://}
+        reference=${uri##*~}
+
+        http_src=${2}
+        checksum=${3}
+        tarball=${4:-$(basename "${http_src}")}
+        if [ "${tarball}" = "_" ]; then
+            echo "${uri}: ERROR! Must have tarball name if no http source."
+            exit 1
+        fi
+        tarball=${dest}/${tarball}
+
+        # Check if tarball already generated + matches checksum
+        checksum=${3}
+        if [ -e "${tarball}" ]; then
+            existing_checksum=$(checksum_file "${tarball}")
+            if [ "${existing_checksum}" = "${checksum}" ]; then
+                return
+            fi
+            rm "${tarball}"
+        fi
+
+        # Clone the repository, or update it if needed
+        if [ ! -e "${outdir}" ]; then
+            mkdir -p "$(dirname "${outdir}")"
+            git clone "${repo}" "${outdir}"
+        elif ! git_ref_exists "${outdir}" "${reference}" ; then
+            (
+                cd "${outdir}" || exit
+                git pull
+            )
+        fi
+
+        # Sanity check: the reference we want exists
+        if ! git_ref_exists "${outdir}" "${reference}"; then
+            echo "${reference} not found in ${repo} (${outdir})"
+            exit 1
+        fi
+
+        # Generate the prefix for the tarball
+        prefix_ref=${reference}
+        # All git repositories we already use remove "v"s from the beginning
+        # of branch/tag names in the tarball prefix
+        if echo "${reference}" | grep -Eq "^v[0-9]"; then
+            prefix_ref=$(echo "${reference}" | sed "s/^v//")
+        fi
+        prefix=$(basename "${repo}" | sed "s/.git$//")-${prefix_ref}
+
+        (
+            cd "${outdir}" || exit
+            git config tar.tar.gz.command gzip
+            # -T1 avoids non-determinism due to threading
+            # This may not be correct for forges other than Savannah
+            git config tar.tar.xz.command "xz -T1"
+            git archive "${reference}" -o "${tarball}" --prefix "${prefix}/"
+        )
+
+        my_checksum=$(sha256sum "${tarball}" | cut -d' ' -f1)
+        if [ "${my_checksum}" != "${checksum}" ]; then
+            echo "${uri}: generated tarball does not match checksum"
+            exit 1
+        fi
+    fi
+}
+
+for src in steps/*/sources; do
+    while read -r line; do
+        # shellcheck disable=SC2086
+        do_file ${line}
+        uri=$(echo "${line}" | cut -d' ' -f1)
+    done < "${src}"
+done
--- a/rootfs.py
+++ b/rootfs.py
@ -16,10 +16,13 @@ you can run bootstap inside chroot.

 import argparse
 import os
+import signal
+import threading

-from lib.utils import run, run_as_root
-from lib.target import Target
 from lib.generator import Generator, stage0_arch_map
+from lib.simple_mirror import SimpleMirror
+from lib.target import Target
+from lib.utils import run, run_as_root

 def create_configuration_file(args):
    """
@ -50,6 +53,12 @@ def create_configuration_file(args):
            config.write("KERNEL_BOOTSTRAP=False\n")
        config.write(f"BUILD_KERNELS={args.update_checksums or args.build_kernels}\n")
        config.write(f"CONFIGURATOR={args.configurator}\n")
+        if not args.external_sources:
+            if args.mirrors:
+                config.write(f"MIRRORS=\"{" ".join(args.mirrors)}\"\n")
+                config.write(f"MIRRORS_LEN={len(args.mirrors)}\n")
+            else:
+                config.write("MIRRORS_LEN=0\n")

 # pylint: disable=too-many-statements,too-many-branches
 def main():
@ -94,6 +103,9 @@ def main():
    parser.add_argument("--configurator",
                        help="Run the interactive configurator",
                        action="store_true")
+    parser.add_argument("-m", "--mirrors",
+                        help="Mirrors to download distfiles from",
+                        nargs='+')
    parser.add_argument("-r", "--repo",
                        help="Path to prebuilt binary packages", nargs=None)
    parser.add_argument("--early-preseed",
@ -168,6 +180,19 @@ def main():
    # Set constant umask
    os.umask(0o022)

+    # file:// mirrors
+    mirror_servers = []
+    if args.mirrors:
+        for i, mirror in enumerate(args.mirrors):
+            if mirror.startswith("file://"):
+                path = mirror.removeprefix("file://")
+                if not path.startswith("/"):
+                    raise ValueError("A file:// mirror must be an absolute path.")
+
+                server = SimpleMirror(path)
+                args.mirrors[i] = f"http://127.0.0.1:{server.port}"
+                mirror_servers.append(server)
+
    # bootstrap.cfg
    try:
        os.remove(os.path.join('steps', 'bootstrap.cfg'))
@ -184,14 +209,25 @@ def main():
    if args.tmpfs:
        target.tmpfs(size=args.tmpfs_size)

+    for server in mirror_servers:
+        thread = threading.Thread(target=server.serve_forever)
+        thread.start()
+
+    def cleanup(*_):
+        for server in mirror_servers:
+            server.shutdown()
+    signal.signal(signal.SIGINT, cleanup)
+
    generator = Generator(arch=args.arch,
                          external_sources=args.external_sources,
                          repo_path=args.repo,
-                          early_preseed=args.early_preseed)
+                          early_preseed=args.early_preseed,
+                          mirrors=args.mirrors)

-    bootstrap(args, generator, target, args.target_size)
+    bootstrap(args, generator, target, args.target_size, cleanup)
+    cleanup()

-def bootstrap(args, generator, target, size):
+def bootstrap(args, generator, target, size, cleanup):
    """Kick off bootstrap process."""
    print(f"Bootstrapping {args.arch}", flush=True)
    if args.chroot:
@ -206,7 +242,8 @@ print(shutil.which('chroot'))

        arch = stage0_arch_map.get(args.arch, args.arch)
        init = os.path.join(os.sep, 'bootstrap-seeds', 'POSIX', arch, 'kaem-optional-seed')
-        run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.target_dir, init)
+        run_as_root('env', '-i', 'PATH=/bin', chroot_binary, generator.target_dir, init,
+                    cleanup=cleanup)

    elif args.bwrap:
        init = '/init'
@ -235,7 +272,8 @@ print(shutil.which('chroot'))
                                  '--proc', '/proc',
                                  '--bind', '/sys', '/sys',
                                  '--tmpfs', '/tmp',
-                                  init)
+                                  init,
+            cleanup=cleanup)

    elif args.bare_metal:
        if args.kernel:
@ -293,7 +331,7 @@ print(shutil.which('chroot'))
            ]
        if not args.interactive:
            arg_list += ['-no-reboot', '-nographic']
-        run(args.qemu_cmd, *arg_list)
+        run(args.qemu_cmd, *arg_list, cleanup=cleanup)

 if __name__ == "__main__":
    main()
--- a/source_manifest.py
+++ b/source_manifest.py
@ -7,8 +7,6 @@ for the bootstrapping process.
 # SPDX-License-Identifier: GPL-3.0-or-later
 # SPDX-FileCopyrightText: 2023 Dor Askayo <dor.askayo@gmail.com>

-import argparse
-
 from lib.generator import Generator

 def main():
--- a/steps/SHA256SUMS.pkgs
+++ b/steps/SHA256SUMS.pkgs
@ -40,13 +40,13 @@ db8757b5fe77f56073caa034bfe9b3af6076d7299db966c4348e06f01a34415f  curl-8.5.0_1.t
 9534d834b018343b8335d0ef0b5faff3ee18e4ecbecf43a2a80deb972283f5d6  dhcpcd-10.0.1_0.tar.bz2
 e6372051f90c40d1f4a5cca2f8930187b363dfc916bcb973c72bca6c0aba3cd5  diffutils-2.7_0.tar.bz2
 41e3129b974391fe4ace69914fdef9b0fe72c6f5d87d495750fb89bdd999fa01  diffutils-3.10_0.tar.bz2
-bda6afcd3a390a34a57443269a4857ccc470129f7a557981778c145fd841cbd1  dist-3.5-236_0.tar.bz2
+283a66575b4d7c835f6ebabb38cfb5d7d62032f832ad949f067a85b8a378c36c  dist-3.5-236_0.tar.bz2
 26c0835a8b9be115082a2866e906363947528169c7274bd3b82752df77e99f11  e2fsprogs-1.45.7_0.tar.bz2
 9c6f7e156f299f1d0948537e28f00d40e727a3debae3cce158e9bce827136214  ed-1.4_0.tar.bz2
 0061d240f9bb2bb534db7003ddcc4eae9981954ad72e8f41ff1d76dc449ebd37  file-5.44_0.tar.bz2
 eec07e0d710ed3dd8d0ba6a1ec6ca9f1fa3a611bbb225195435679175317bf90  findutils-4.2.33_0.tar.bz2
 80798628ccb38521f42e1ca1abe4f0244e4b283e6f662cd354ab70e61a9b4675  flex-2.5.11_0.tar.bz2
-b3a8d59229289ca5a1de6a24c9c828d2847144f6662e0d5b7f41e7ff6e57abaa  flex-2.5.33_0.tar.bz2
+62b4ba8e73daf1f4ac4fc1f90e99394e40d7c03dce9fed8539dbc01d024c62e8  flex-2.5.33_0.tar.bz2
 e10060b55e2378a065cf8590cae17400c8befc6b3bb0cbbb7093f1b9f46ec4cc  flex-2.6.4_0.tar.bz2
 5619cf9a893902491a9d8a8f8da4d39efb3606d7d0a72cb4aaa9d1ff798b5495  gawk-3.0.4_0.tar.bz2
 62e90abe883f6ccd679adf42d00c2c18a9809bbab599411c81b6c4ac4d446452  gawk-5.3.0_0.tar.bz2
@ -97,7 +97,7 @@ a18c4b2e5de2bfe5bb3ee9d360484fcfebad3df042f1859d4aa333dd60f55e56  opendoas-6.8.2
 287b0a8c0de14a2817e68ab023bfc502a0880d8c34386162351d5ffaf942da70  perl-5.000_0.tar.bz2
 9617799cdc9d4be39d31af54b9bcae47941d68416e9cc9c92cbc2ed08785dbc1  perl-5.003_0.tar.bz2
 74d64a8af080022432fa94dba449090419d25b103d247710dc0b6102a4ad86a6  perl-5.10.1_0.tar.bz2
-d381e910532061ac97e48ff09906bddbcb0956b37fc47d8c666f56a56b50c589  perl-5.32.1_0.tar.bz2
+5321a3a3bae7d689b085b415f4f445ebf19b801514340879a2b4c1bf97f05533  perl-5.32.1_0.tar.bz2
 d63d2fda752759778ae2d0dd08e6c023a81afbfc63ce11db860fc6858ae26c7f  perl-5.6.2_0.tar.bz2
 c39ab70bce0bd09f013109ad7370085e90d77ef2b55abcfdc922d7832524c9cf  perl5.004-05_0.tar.bz2
 2ac211ad3cf966dd89434c175970e10699c363b66228abc1642766aeafe2766a  perl5.005-03_0.tar.bz2
--- a/steps/autogen-5.18.16/sources
+++ b/steps/autogen-5.18.16/sources
@ -1,6 +1,4 @@
-https://github.com/schierlm/gnu-autogen-bootstrapping/archive/refs/tags/autogen-5.18.16-v1.0.1.tar.gz 953ba180b18acff188a0a8700770c7cf2fc97e1683c7b9699a5a748b542ccdd5
+git://github.com/schierlm/gnu-autogen-bootstrapping~autogen-5.18.16-v1.0.1 https://github.com/schierlm/gnu-autogen-bootstrapping/archive/refs/tags/autogen-5.18.16-v1.0.1.tar.gz 953ba180b18acff188a0a8700770c7cf2fc97e1683c7b9699a5a748b542ccdd5
 https://mirrors.kernel.org/gnu/autogen/rel5.18.16/autogen-5.18.16.tar.xz f8a13466b48faa3ba99fe17a069e71c9ab006d9b1cfabe699f8c60a47d5bb49a
-https://git.savannah.gnu.org/cgit/autogen.git/snapshot/autogen-5.18.16.tar.gz 0c04ab2f7ce13c4a1c06c4abc7dfe75312aad89b8b0a1068e5e563787eb56632
-https://files.bootstrapping.world/autogen-5.18.16.tar.gz 0c04ab2f7ce13c4a1c06c4abc7dfe75312aad89b8b0a1068e5e563787eb56632
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab
-https://files.bootstrapping.world/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab
+git://git.savannah.gnu.org/autogen.git~v5.18.16 https://git.savannah.gnu.org/cgit/autogen.git/snapshot/autogen-5.18.16.tar.gz 0c04ab2f7ce13c4a1c06c4abc7dfe75312aad89b8b0a1068e5e563787eb56632
+git://git.savannah.gnu.org/gnulib.git~8f4538a5 _ e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab gnulib-8f4538a5.tar.gz
--- a/steps/bash-2.05b/pass1.kaem
+++ b/steps/bash-2.05b/pass1.kaem
@ -16,8 +16,8 @@ mkdir build src
 cd build

 # Extract
-cp ${DISTFILES}/${pkg}.tar.bz2 ../src/
-bzip2 -d -f ../src/${pkg}.tar.bz2
+cp ${DISTFILES}/${pkg}.tar.gz ../src/
+gzip -d -f ../src/${pkg}.tar.gz
 tar xf ../src/${pkg}.tar
 rm -r ../src/
 cd ${pkg}
--- a/steps/bash-2.05b/sources
+++ b/steps/bash-2.05b/sources
@ -1 +1 @@
-https://src.fedoraproject.org/repo/pkgs/bash/bash-2.05b.tar.bz2/f3e5428ed52a4f536f571a945d5de95d/bash-2.05b.tar.bz2 1ce4e5b47a6354531389f0adefb54dee2823227bf6e1e59a31c0e9317a330822
+https://mirrors.kernel.org/gnu/bash/bash-2.05b.tar.gz ba03d412998cc54bd0b0f2d6c32100967d3137098affdc2d32e6e7c11b163fe4
--- a/steps/bc-1.07.1/sources
+++ b/steps/bc-1.07.1/sources
@ -1 +1 @@
-https://mirrors.kernel.org/slackware/slackware64-15.0/source/ap/bc/bc-1.07.1.tar.xz 95396f88fc719a1bf8bd463809119526fef44a42ab9eb708335c2cb79bc801c6
+https://mirrors.kernel.org/gnu/bc/bc-1.07.1.tar.gz 62adfca89b0a1c0164c2cdca59ca210c1d44c3ffc46daf9931cf4942664cb02a
--- a/steps/bison-2.3/sources
+++ b/steps/bison-2.3/sources
@ -1,3 +1,2 @@
 http://mirrors.kernel.org/gnu/bison/bison-2.3.tar.bz2 b10d7e9e354be72aee4e4911cf19dd27b5c527d4e7200857365b5fcdeea0dffb
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-b28236b.tar.gz 0190f28cb155fedd22bf8558c3e8705eed9eacfb7ae29e7508d025a68eb90899
-https://files.bootstrapping.world/gnulib-b28236b.tar.gz 0190f28cb155fedd22bf8558c3e8705eed9eacfb7ae29e7508d025a68eb90899
+git://git.savannah.gnu.org/gnulib.git~b28236b _ 0190f28cb155fedd22bf8558c3e8705eed9eacfb7ae29e7508d025a68eb90899 gnulib-b28236b.tar.gz
--- a/steps/bison-3.4.2/sources
+++ b/steps/bison-3.4.2/sources
@ -1,3 +1,2 @@
 http://mirrors.kernel.org/gnu/bison/bison-3.4.2.tar.xz 27d05534699735dc69e86add5b808d6cb35900ad3fd63fa82e3eb644336abfa0
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-672663a.tar.gz 8cced51f89a950472473856f86e88f5daf97a2347756125ccdc8ee907deec570
-https://files.bootstrapping.world/gnulib-672663a.tar.gz 8cced51f89a950472473856f86e88f5daf97a2347756125ccdc8ee907deec570
+git://git.savannah.gnu.org/gnulib.git~672663a _ 8cced51f89a950472473856f86e88f5daf97a2347756125ccdc8ee907deec570 gnulib-672663a.tar.gz
--- a/steps/bzip2-1.0.8/pass1.kaem
+++ b/steps/bzip2-1.0.8/pass1.kaem
@ -15,8 +15,8 @@ mkdir build src
 cd build

 # Extract
-cp ${DISTFILES}/${pkg}.tar.xz ../src/
-unxz --file ../src/${pkg}.tar.xz --output ../src/${pkg}.tar
+cp ${DISTFILES}/${pkg}.tar.gz ../src/
+gzip -d -f ../src/${pkg}.tar.gz
 tar xf ../src/${pkg}.tar
 rm -r ../src
 cd ${pkg}
--- a/steps/bzip2-1.0.8/sources
+++ b/steps/bzip2-1.0.8/sources
@ -1 +1 @@
-https://mirrors.kernel.org/slackware/slackware-14.0/patches/source/bzip2/bzip2-1.0.8.tar.xz 47fd74b2ff83effad0ddf62074e6fad1f6b4a77a96e121ab421c20a216371a1f
+https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
--- a/steps/coreutils-9.4/sources
+++ b/steps/coreutils-9.4/sources
@ -1,7 +1,5 @@
-http://git.savannah.gnu.org/cgit/coreutils.git/snapshot/coreutils-9.4.tar.xz 8fb56810310253300b3d6f84e68dc97eb2d74e1f4f78e05776831d9d82e4f2d7
-https://files.bootstrapping.world/coreutils-9.4.tar.xz 8fb56810310253300b3d6f84e68dc97eb2d74e1f4f78e05776831d9d82e4f2d7
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-bb5bb43.tar.gz b8aa1ac1b18c67f081486069e6a7a5564f20431c2313a94c20a46dcfb904be2a
-https://files.bootstrapping.world/gnulib-bb5bb43.tar.gz b8aa1ac1b18c67f081486069e6a7a5564f20431c2313a94c20a46dcfb904be2a
+git://git.savannah.gnu.org/coreutils.git~v9.4 http://git.savannah.gnu.org/cgit/coreutils.git/snapshot/coreutils-9.4.tar.xz 8fb56810310253300b3d6f84e68dc97eb2d74e1f4f78e05776831d9d82e4f2d7
+git://git.savannah.gnu.org/gnulib.git~bb5bb43 _ b8aa1ac1b18c67f081486069e6a7a5564f20431c2313a94c20a46dcfb904be2a gnulib-bb5bb43.tar.gz
 http://ftp.unicode.org/Public/15.0.0/ucd/UnicodeData.txt 806e9aed65037197f1ec85e12be6e8cd870fc5608b4de0fffd990f689f376a73 UnicodeData-15.0.0.txt
 http://ftp.unicode.org/Public/15.0.0/ucd/PropList.txt e05c0a2811d113dae4abd832884199a3ea8d187ee1b872d8240a788a96540bfd PropList-15.0.0.txt
 http://ftp.unicode.org/Public/15.0.0/ucd/DerivedCoreProperties.txt d367290bc0867e6b484c68370530bdd1a08b6b32404601b8c7accaf83e05628d DerivedCoreProperties-15.0.0.txt
--- a/steps/diffutils-3.10/sources
+++ b/steps/diffutils-3.10/sources
@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/diffutils/diffutils-3.10.tar.xz 90e5e93cc724e4ebe12ede80df1634063c7a855692685919bfe60b556c9bd09e
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-5d2fe24.tar.gz 72e7bb2d1d75e63d1c46d33b8dd22e8eb60afdba4af3e7251151b5c2a6f00bfb
-https://files.bootstrapping.world/gnulib-5d2fe24.tar.gz 72e7bb2d1d75e63d1c46d33b8dd22e8eb60afdba4af3e7251151b5c2a6f00bfb
+git://git.savannah.gnu.org/gnulib.git~5d2fe24 _ 72e7bb2d1d75e63d1c46d33b8dd22e8eb60afdba4af3e7251151b5c2a6f00bfb gnulib-5d2fe24.tar.gz
--- a/steps/dist-3.5-236/patches/env.patch
+++ b/steps/dist-3.5-236/patches/env.patch
@ -4,8 +4,8 @@ SPDX-FileCopyrightText: 2021 fosslinux <fosslinux@aussies.space>

 SPDX-License-Identifier: Artistic-1.0

--- dist-3.5-236/bin/perload	2021-04-21 19:01:35.803767498 +1000
-+++ dist-3.5-236/bin/perload	2021-04-21 19:01:47.959850217 +1000
+--- dist-2cec35331a912b165e2dd135d22de81f34bbc83f/bin/perload	2021-04-21 19:01:35.803767498 +1000
+++ dist-2cec35331a912b165e2dd135d22de81f34bbc83f/bin/perload	2021-04-21 19:01:47.959850217 +1000
@@ -450,8 +450,6 @@
 :# This perl program uses dynamic loading [generated by perload]
 :#
--- a/steps/dist-3.5-236/sources
+++ b/steps/dist-3.5-236/sources
@ -1 +1 @@
-http://deb.debian.org/debian/pool/main/d/dist/dist_3.5-236.orig.tar.gz 05fa4f6ea9f05adf8f577699cb3f5b88b20dfce86b0d0cebbfb072fe5933d38f
+git://github.com/rmanfredi/dist~2cec35331a912b165e2dd135d22de81f34bbc83f https://github.com/rmanfredi/dist/archive/2cec35331a912b165e2dd135d22de81f34bbc83f.tar.gz 6d2c9e953de2c136c77c9b6485fbc61e8291a2a70689f2a07c79d9381bf9dbcb
--- a/steps/ed-1.4/sources
+++ b/steps/ed-1.4/sources
@ -1 +1 @@
-https://mirrors.kernel.org/slackware/slackware-13.37/source/a/ed/ed-1.4.tar.xz 102c80e6da527c6b8eebd5195cd05fc71808d60735d73e8bb503a5e294475007
+https://mirrors.kernel.org/gnu/ed/ed-1.4.tar.gz db36da85ee1a9d8bafb4b041bd4c8c11becba0c43ec446353b67045de1634fda
--- a/steps/findutils-4.2.33/pass1.sh
+++ b/steps/findutils-4.2.33/pass1.sh
@ -3,8 +3,6 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later

-EXTRA_DISTFILES="gnulib-8e128e.tar.gz"
-
 src_prepare() {
    . ../../import-gnulib.sh

--- a/steps/findutils-4.2.33/sources
+++ b/steps/findutils-4.2.33/sources
@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/findutils/findutils-4.2.33.tar.gz 813cd9405aceec5cfecbe96400d01e90ddad7b512d3034487176ce5258ab0f78
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8e128e.tar.gz 0cfbf866bc39c31f25fa0e56af1e56c5e5c92fc1e5d51242ebafef7ea211f3d5
-https://files.bootstrapping.world/gnulib-8e128e.tar.gz 0cfbf866bc39c31f25fa0e56af1e56c5e5c92fc1e5d51242ebafef7ea211f3d5
+git://git.savannah.gnu.org/gnulib.git~8e128e _ 0cfbf866bc39c31f25fa0e56af1e56c5e5c92fc1e5d51242ebafef7ea211f3d5 gnulib-8e128e.tar.gz
--- a/steps/flex-2.5.11/pass1.sh
+++ b/steps/flex-2.5.11/pass1.sh
@ -8,7 +8,6 @@ src_prepare() {
    default

    touch config.h
-    rm parse.c parse.h scan.c skel.c
 }

 src_compile() {
--- a/steps/flex-2.5.11/patches/scan_l.patch
+++ b/steps/flex-2.5.11/patches/scan_l.patch
@ -8,8 +8,8 @@ Comments are unsupported by our flex.

 diff --git scan.l scan.l
 index 18d0de8..c251a5e 100644
--- flex-2.5.11/scan.l
-+++ flex-2.5.11/scan.l
+--- flex-d160f0247ba1611aa59d28f027d6292ba24abb50/scan.l
+++ flex-d160f0247ba1611aa59d28f027d6292ba24abb50/scan.l
@@ -335,8 +335,8 @@ LEXOPT		[aceknopr]
 
 
--- a/steps/flex-2.5.11/patches/yyin.patch
+++ b/steps/flex-2.5.11/patches/yyin.patch
@ -9,8 +9,8 @@ acidentally re-declare it.

 diff --git flexdef.h flexdef.h
 index 3eb710a..94ef024 100644
--- flex-2.5.11/flexdef.h
-+++ flex-2.5.11/flexdef.h
+--- flex-d160f0247ba1611aa59d28f027d6292ba24abb50/flexdef.h
+++ flex-d160f0247ba1611aa59d28f027d6292ba24abb50/flexdef.h
@@ -421,7 +421,7 @@ extern int yymore_really_used, reject_really_used;
  */
 
@ -31,8 +31,8 @@ index 3eb710a..94ef024 100644
 extern char *action_array;
 extern int action_size;
 extern int defs1_offset, prolog_offset, action_offset, action_index;
--- flex-2.5.11/scan.l
-+++ flex-2.5.11/scan.l
+--- flex-d160f0247ba1611aa59d28f027d6292ba24abb50/scan.l
+++ flex-d160f0247ba1611aa59d28f027d6292ba24abb50/scan.l
@@ -32,6 +32,7 @@
 /*  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR */
 /*  PURPOSE. */
--- a/steps/flex-2.5.11/sources
+++ b/steps/flex-2.5.11/sources
@ -1 +1 @@
-http://ftp-tel.sjtu.edu.cn/sites/ftp.openpkg.org/sources/DST/flex/flex-2.5.11.tar.gz bc79b890f35ca38d66ff89a6e3758226131e51ccbd10ef78d5ff150b7bd73689
+git://github.com/westes/flex~d160f0247ba1611aa59d28f027d6292ba24abb50 https://github.com/westes/flex/archive/d160f0247ba1611aa59d28f027d6292ba24abb50.tar.gz 68aa10c473b6010ffad680cada09fc4eec6b3cc6e415cc2339e5fc2385ccc142
--- a/steps/flex-2.5.33/pass1.sh
+++ b/steps/flex-2.5.33/pass1.sh
@ -6,12 +6,6 @@ src_prepare() {
    default

    AUTOPOINT=true AUTOMAKE=automake-1.15 ACLOCAL=aclocal-1.15 autoreconf-2.69 -fi
-
-    # Remove pregenerated files
-    rm parse.c parse.h scan.c 
-
-    # Remove pregenerated .info
-    rm doc/flex.info
 }

 src_configure() {
--- a/steps/flex-2.5.33/patches/disable-unavailables.patch
+++ b/steps/flex-2.5.33/patches/disable-unavailables.patch
@ -0,0 +1,50 @@
+Disable things we can't regenerate for varied reasons.
+
+SPDX-FileCopyrightText: 2025 fosslinux <fosslinux@aussies.space>
+
+SPDX-License-Identifier: BSD-3-Clause-flex
+
+diff --color -ru configure.in
+--- flex-e6f147b7a5f2ec2dc862dc9d30b3734b9555a1ea/configure.in	2024-12-28 11:29:45.497750429 +1100
+++ flex-e6f147b7a5f2ec2dc862dc9d30b3734b9555a1ea/configure.in	2024-12-28 12:37:00.190457545 +1100
+@@ -31,9 +31,6 @@
+ 
+ dnl checks for programs
+ 
+-AM_GNU_GETTEXT([external])
+-AM_GNU_GETTEXT_VERSION(0.12)
+-
+ AC_PROG_YACC
+ AM_PROG_LEX
+ AC_PROG_CC
+@@ -97,12 +94,10 @@
+ flex.spec
+ Makefile
+ doc/Makefile
+-website/Makefile
+ examples/Makefile
+ examples/fastwc/Makefile
+ examples/manual/Makefile
+ m4/Makefile
+-po/Makefile.in
+ tools/Makefile
+ tests/Makefile
+ tests/TEMPLATE/Makefile
+diff --color -ru Makefile.am
+--- flex-e6f147b7a5f2ec2dc862dc9d30b3734b9555a1ea/Makefile.am	2024-12-28 11:29:45.496750429 +1100
+++ flex-e6f147b7a5f2ec2dc862dc9d30b3734b9555a1ea/Makefile.am	2024-12-28 13:32:54.945214018 +1100
+@@ -135,13 +135,12 @@
+ 	doc \
+ 	m4  \
+ 	examples \
+-	po \
+ 	tools \
+ 	tests
+ 
+ localedir = $(datadir)/locale
+ AM_CPPFLAGS = -DLOCALEDIR=\"$(localedir)\" -I@includedir@ -I$(top_srcdir)/intl
+-LIBS = @LIBINTL@ @LIBS@
+LIBS = @LIBS@
+ 
+ skel.c: flex.skl mkskel.sh flexint.h tables_shared.h
+ 	sed 's/m4_/m4postproc_/g; s/m4preproc_/m4_/g' $(srcdir)/flex.skl | $(m4) -P -DFLEX_MAJOR_VERSION=`echo $(VERSION)|cut -f 1 -d .` -DFLEX_MINOR_VERSION=`echo $(VERSION)|cut -f 2 -d .` -DFLEX_SUBMINOR_VERSION=`echo $(VERSION)|cut -f 3 -d .` | sed 's/m4postproc_/m4_/g' | $(SHELL) $(srcdir)/mkskel.sh  >skel.c
--- a/steps/flex-2.5.33/sources
+++ b/steps/flex-2.5.33/sources
@ -1 +1 @@
-http://ftp-tel.sjtu.edu.cn/sites/ftp.openpkg.org/sources/DST/flex/flex-2.5.33.tar.gz c40385e142989c91989413f3c5a31282b2ffdca16b69cd3ecfde537b8a474921
+git://github.com/westes/flex~e6f147b7a5f2ec2dc862dc9d30b3734b9555a1ea https://github.com/westes/flex/archive/e6f147b7a5f2ec2dc862dc9d30b3734b9555a1ea.tar.gz baec69069ff58b7cdbe0103ffc16f29d4857428c29efcdf685c574d8300fd838
--- a/steps/gcc-10.4.0/sources
+++ b/steps/gcc-10.4.0/sources
@ -1 +1 @@
-http://ftp.gnu.org/gnu/gcc/gcc-10.4.0/gcc-10.4.0.tar.xz c9297d5bcd7cb43f3dfc2fed5389e948c9312fd962ef6a4ce455cff963ebe4f1
+http://mirrors.kernel.org/gnu/gcc/gcc-10.4.0/gcc-10.4.0.tar.xz c9297d5bcd7cb43f3dfc2fed5389e948c9312fd962ef6a4ce455cff963ebe4f1
--- a/steps/gcc-13.1.0/sources
+++ b/steps/gcc-13.1.0/sources
@ -1 +1 @@
-https://ftp.gnu.org/gnu/gcc/gcc-13.1.0/gcc-13.1.0.tar.xz 61d684f0aa5e76ac6585ad8898a2427aade8979ed5e7f85492286c4dfc13ee86
+https://mirrors.kernel.org/gnu/gcc/gcc-13.1.0/gcc-13.1.0.tar.xz 61d684f0aa5e76ac6585ad8898a2427aade8979ed5e7f85492286c4dfc13ee86
--- a/steps/gettext-0.21/sources
+++ b/steps/gettext-0.21/sources
@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/gettext/gettext-0.21.tar.xz d20fcbb537e02dcf1383197ba05bd0734ef7bf5db06bdb241eb69b7d16b73192
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-7daa86f.tar.gz 2d911c2f2ed97b347d6d360b742abdc98aa626d4f8f847ee682c7cde12e90871
-https://files.bootstrapping.world/gnulib-7daa86f.tar.gz 2d911c2f2ed97b347d6d360b742abdc98aa626d4f8f847ee682c7cde12e90871
+git://git.savannah.gnu.org/gnulib.git~7daa86f _ 2d911c2f2ed97b347d6d360b742abdc98aa626d4f8f847ee682c7cde12e90871 gnulib-7daa86f.tar.gz
--- a/steps/grep-3.7/pass1.sh
+++ b/steps/grep-3.7/pass1.sh
@ -2,9 +2,6 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later

-urls="https://mirrors.kernel.org/gnu/grep/grep-3.7.tar.xz
- http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8f4538a5.tar.gz"
-
 src_prepare() {
    rm configure
    find . -name 'Makefile.in' -delete
--- a/steps/grep-3.7/sources
+++ b/steps/grep-3.7/sources
@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/grep/grep-3.7.tar.xz 5c10da312460aec721984d5d83246d24520ec438dd48d7ab5a05dbc0d6d6823c
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab
-https://files.bootstrapping.world/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab
+git://git.savannah.gnu.org/gnulib.git~8f4538a5 _ e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab gnulib-8f4538a5.tar.gz
--- a/steps/grub-2.06/sources
+++ b/steps/grub-2.06/sources
@ -1,6 +1,5 @@
 https://mirrors.kernel.org/gnu/grub/grub-2.06.tar.xz b79ea44af91b93d17cd3fe80bdae6ed43770678a9a5ae192ccea803ebb657ee1
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-d271f86.tar.gz 31d69d3d251e39135b5194ddc6f897910d344059f7494d96a739aecbf7ac2b66
-https://files.bootstrapping.world/gnulib-d271f86.tar.gz 31d69d3d251e39135b5194ddc6f897910d344059f7494d96a739aecbf7ac2b66
+git://git.savannah.gnu.org/gnulib.git~d271f86 _ 31d69d3d251e39135b5194ddc6f897910d344059f7494d96a739aecbf7ac2b66 gnulib-d271f86.tar.gz
 http://ftp.unicode.org/Public/9.0.0/ucd/UnicodeData.txt 68dfc414d28257b9b5d6ddbb8b466c768c00ebdf6cbf7784364a9b6cad55ee8f UnicodeData-9.0.0.txt
 http://ftp.unicode.org/Public/9.0.0/ucd/PropList.txt f413ea8dbd3858de72f3148b47dd0586019761357d1481e3b65f3a025bc27f82 PropList-9.0.0.txt
 http://ftp.unicode.org/Public/9.0.0/ucd/DerivedCoreProperties.txt 6662c7e30b572df5d948c092692f52bcc79ab36d49a063a73d6435042db6fb3b DerivedCoreProperties-9.0.0.txt
--- a/steps/guile-3.0.9/sources
+++ b/steps/guile-3.0.9/sources
@ -1,7 +1,5 @@
 https://mirrors.kernel.org/gnu/guile/guile-3.0.7.tar.xz f57d86c70620271bfceb7a9be0c81744a033f08adc7ceba832c9917ab3e691b7
 https://mirrors.kernel.org/gnu/guile/guile-3.0.9.tar.xz 1a2625ac72b2366e95792f3fe758fd2df775b4044a90a4a9787326e66c0d750d 
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-901694b9.tar.gz f9aad85de1f41d57c9368d304020ffbf354a5e56db1297f022c3d12181134e56
-https://files.bootstrapping.world/gnulib-901694b9.tar.gz f9aad85de1f41d57c9368d304020ffbf354a5e56db1297f022c3d12181134e56
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-356a414e.tar.gz fc9973f1a9243fdc4b98d33d7704f3c71bfdc4c2ef96899b8f28cade7290a714
-https://files.bootstrapping.world/gnulib-356a414e.tar.gz fc9973f1a9243fdc4b98d33d7704f3c71bfdc4c2ef96899b8f28cade7290a714
-https://github.com/schierlm/guile-psyntax-bootstrapping/archive/refs/tags/guile-3.0.7.tar.gz 14cda9c416506dfadf60c14fc623ff01ef99b87564a78d0a29c5d17143c97609
+git://git.savannah.gnu.org/gnulib.git~901694b9 _ f9aad85de1f41d57c9368d304020ffbf354a5e56db1297f022c3d12181134e56 gnulib-901694b9.tar.gz
+git://git.savannah.gnu.org/gnulib.git~356a414e _ fc9973f1a9243fdc4b98d33d7704f3c71bfdc4c2ef96899b8f28cade7290a714 gnulib-356a414e.tar.gz
+git://github.com/schierlm/guile-psyntax-bootstrapping~guile-3.0.7 https://github.com/schierlm/guile-psyntax-bootstrapping/archive/refs/tags/guile-3.0.7.tar.gz 14cda9c416506dfadf60c14fc623ff01ef99b87564a78d0a29c5d17143c97609
--- a/steps/gzip-1.13/sources
+++ b/steps/gzip-1.13/sources
@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/gzip/gzip-1.13.tar.xz 7454eb6935db17c6655576c2e1b0fabefd38b4d0936e0f87f48cd062ce91a057
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-5651802.tar.gz 56f1221eb682c3502ee097f583f44673570753cb452346ad4806d94560c3fac9
-https://files.bootstrapping.world/gnulib-5651802.tar.gz 56f1221eb682c3502ee097f583f44673570753cb452346ad4806d94560c3fac9
+git://git.savannah.gnu.org/gnulib.git~5651802 _ 56f1221eb682c3502ee097f583f44673570753cb452346ad4806d94560c3fac9 gnulib-5651802.tar.gz
--- a/steps/helpers.sh
+++ b/steps/helpers.sh
@ -21,8 +21,6 @@ _get_files() {
    prefix="${1}"
    fs=
    if [ -n "$(ls 2>/dev/null)" ]; then
-        # This can be removed once Debian 12 is stable
-        # shellcheck disable=SC2035
        fs=$(echo *)
    fi
    if [ -n "$(ls .[0-z]* 2>/dev/null)" ]; then
@ -258,18 +256,48 @@ build() {
    unset extract
 }

+# An inventive way to randomise with what we know we always have
+randomize() {
+    if command -v shuf >/dev/null 2>&1; then
+        # shellcheck disable=SC2086
+        shuf -e ${1} | tr '\n' ' '
+    else
+        mkdir -p /tmp/random
+        for item in ${1}; do
+            touch --date=@${RANDOM} /tmp/random/"${item//\//~*~*}"
+        done
+        # cannot rely on find existing
+        # shellcheck disable=SC2012
+        ls -1 -t /tmp/random | sed 's:~\*~\*:/:g' | tr '\n' ' '
+        rm -r /tmp/random
+    fi
+}
+
 download_source_line() {
-    url="${1}"
+    if [[ "${1}" == git://* ]]; then
+        shift
+    fi
+    upstream_url="${1}"
    checksum="${2}"
    fname="${3}"
    # Default to basename of url if not given
-    fname="${fname:-$(basename "${url}")}"
+    fname="${fname:-$(basename "${upstream_url}")}"
    if ! [ -e "${fname}" ]; then
-        curl --fail --retry 5 --location "${url}" --output "${fname}" || true
+        for mirror in $(randomize "${MIRRORS}"); do
+            mirror_url="${mirror}/${fname}"
+            echo "${mirror_url}"
+            curl --fail --retry 3 --location "${mirror_url}" --output "${fname}" || true && break
+        done
+        if ! [ -e "${fname}" ] && [ "${upstream_url}" != "_" ]; then
+            curl --fail --retry 3 --location "${upstream_url}" --output "${fname}" || true
+        fi
    fi
 }

 check_source_line() {
+    if [[ "${1}" == git://* ]]; then
+        shift
+    fi
    url="${1}"
    checksum="${2}"
    fname="${3}"
@ -304,6 +332,9 @@ default_src_get() {

 # Intelligently extracts a file based upon its filetype.
 extract_file() {
+    if [[ "${1}" == git://* ]]; then
+        shift
+    fi
    f="${3:-$(basename "${1}")}"
    # shellcheck disable=SC2154
    case "${noextract}" in
--- a/steps/improve/setup_repo.sh
+++ b/steps/improve/setup_repo.sh
@ -6,4 +6,4 @@
 #
 mkdir -p /external/repo

-tar -cf - --exclude='/external/repo/*' --exclude='/external/repo-preseeded/*' --exclude='/external/distfiles/*' --exclude='/dev/*' --exclude='/proc/*' --exclude='/sys/*' --exclude='/tmp/*' / | bzip2 --best > /external/repo/base.tar.bz2
+#tar -cf - --exclude='/external/repo/*' --exclude='/external/repo-preseeded/*' --exclude='/external/distfiles/*' --exclude='/dev/*' --exclude='/proc/*' --exclude='/sys/*' --exclude='/tmp/*' / | bzip2 --best > /external/repo/base.tar.bz2
--- a/steps/kexec-tools-2.0.22/sources
+++ b/steps/kexec-tools-2.0.22/sources
@ -1 +1 @@
-https://github.com/horms/kexec-tools/archive/refs/tags/v2.0.22.tar.gz af618de7848142f204b57811f703de3ae7aa3f5bc5d52226db35800fa8fc4dff
+git://github.com/horms/kexec-tools~v2.0.22 https://github.com/horms/kexec-tools/archive/refs/tags/v2.0.22.tar.gz af618de7848142f204b57811f703de3ae7aa3f5bc5d52226db35800fa8fc4dff
--- a/steps/libtool-2.4.7/sources
+++ b/steps/libtool-2.4.7/sources
@ -1,3 +1,2 @@
 http://mirrors.kernel.org/gnu/libtool/libtool-2.4.7.tar.xz 4f7f217f057ce655ff22559ad221a0fd8ef84ad1fc5fcb6990cecc333aa1635d
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-a521820.tar.gz 719b399fe09a8f6ca14ba8c4a9a60ce9f93f4892effb50961ef3d8cd1a33ff65
-https://files.bootstrapping.world/gnulib-a521820.tar.gz 719b399fe09a8f6ca14ba8c4a9a60ce9f93f4892effb50961ef3d8cd1a33ff65
+git://git.savannah.gnu.org/gnulib.git~a521820 _ 719b399fe09a8f6ca14ba8c4a9a60ce9f93f4892effb50961ef3d8cd1a33ff65 gnulib-a521820.tar.gz
--- a/steps/libunistring-0.9.10/sources
+++ b/steps/libunistring-0.9.10/sources
@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/libunistring/libunistring-0.9.10.tar.xz eb8fb2c3e4b6e2d336608377050892b54c3c983b646c561836550863003c05d7
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-52a06cb3.tar.gz 009989b81c0bebc5f6550636ed653fbcb237dafc2af5c706f3522087ca571e4d
-https://files.bootstrapping.world/gnulib-52a06cb3.tar.gz 009989b81c0bebc5f6550636ed653fbcb237dafc2af5c706f3522087ca571e4d
+git://git.savannah.gnu.org/gnulib.git~52a06cb3 _ 009989b81c0bebc5f6550636ed653fbcb237dafc2af5c706f3522087ca571e4d gnulib-52a06cb3.tar.gz
--- a/steps/m4-1.4.19/sources
+++ b/steps/m4-1.4.19/sources
@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/m4/m4-1.4.19.tar.xz 63aede5c6d33b6d9b13511cd0be2cac046f2e70fd0a07aa9573a04a82783af96
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-3639c57.tar.gz 97dfbad67832641bc7f73437617b78abeafb9946723f19cf4c2ceecfc65fa48d
-https://files.bootstrapping.world/gnulib-3639c57.tar.gz 97dfbad67832641bc7f73437617b78abeafb9946723f19cf4c2ceecfc65fa48d
+git://git.savannah.gnu.org/gnulib.git~3639c57 _ 97dfbad67832641bc7f73437617b78abeafb9946723f19cf4c2ceecfc65fa48d gnulib-3639c57.tar.gz
--- a/steps/openssl-3.0.13/sources
+++ b/steps/openssl-3.0.13/sources
@ -1 +1 @@
-http://snapshot.debian.org/archive/debian/20240325T091329Z/pool/main/o/openssl/openssl_3.0.13.orig.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313
+https://github.com/openssl/openssl/releases/download/openssl-3.0.13/openssl-3.0.13.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313
--- a/steps/patch-2.7.6/sources
+++ b/steps/patch-2.7.6/sources
@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/patch/patch-2.7.6.tar.xz ac610bda97abe0d9f6b7c963255a11dcb196c25e337c61f94e4778d632f1d8fd
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-e017871.tar.gz a285dc300c3d9c25cc06e38827ef40f6073ec3b9b0fcb5bba433f943be92d8d4
-https://files.bootstrapping.world/gnulib-e017871.tar.gz a285dc300c3d9c25cc06e38827ef40f6073ec3b9b0fcb5bba433f943be92d8d4
+git://git.savannah.gnu.org/gnulib.git~e017871 _ a285dc300c3d9c25cc06e38827ef40f6073ec3b9b0fcb5bba433f943be92d8d4 gnulib-e017871.tar.gz
--- a/steps/perl-5.32.1/pass1.sh
+++ b/steps/perl-5.32.1/pass1.sh
@ -24,8 +24,8 @@ src_prepare() {

    # Regenerate configure + config_h.SH
    rm -f Configure config_h.SH
-    ln -s ../metaconfig-5.32.1\~rc1/.package .
-    ln -s ../metaconfig-5.32.1\~rc1/U .
+    ln -s ../metaconfig-5.32.1/.package .
+    ln -s ../metaconfig-5.32.1/U .
    metaconfig -m
 }

--- a/steps/perl-5.32.1/patches/reproducibility.patch
+++ b/steps/perl-5.32.1/patches/reproducibility.patch
@ -7,8 +7,8 @@ generate with correct values for live-bootstrap.

 NOTE: this patch CANNOT be applied to a non-live-bootstrap environment.

--- metaconfig-5.32.1~rc1/U/threads/archname.U	2022-02-27 21:30:03.155396204 +1100
-+++ metaconfig-5.32.1~rc1/U/threads/archname.U	2022-02-27 21:30:49.392396204 +1100
+--- metaconfig-5.32.1/U/threads/archname.U	2022-02-27 21:30:03.155396204 +1100
+++ metaconfig-5.32.1/U/threads/archname.U	2022-02-27 21:30:49.392396204 +1100
@@ -79,7 +79,7 @@
 ?X: Very GCCian.
 *)  archname=`echo $targetarch|sed 's,^[^-]*-,,'` ;;
@ -18,8 +18,8 @@ NOTE: this patch CANNOT be applied to a non-live-bootstrap environment.
 case "$archname" in
 '') dflt="$tarch";;
 *) dflt="$archname";;
--- metaconfig-5.32.1~rc1/U/modified/Oldconfig.U	2022-02-27 21:31:26.911396204 +1100
-+++ metaconfig-5.32.1~rc1/U/modified/Oldconfig.U	2022-02-27 21:32:31.846396204 +1100
+--- metaconfig-5.32.1/U/modified/Oldconfig.U	2022-02-27 21:31:26.911396204 +1100
+++ metaconfig-5.32.1/U/modified/Oldconfig.U	2022-02-27 21:32:31.846396204 +1100
@@ -117,16 +117,13 @@
 ?X: on some machines to avoid the error message when uname is not found; e.g.
 ?X: old SUN-OS 3.2 would not execute hostname in (uname -a || hostname). Sigh!
--- a/steps/perl-5.32.1/patches/reproducibility2.patch
+++ b/steps/perl-5.32.1/patches/reproducibility2.patch
@ -7,8 +7,8 @@ generate with correct values for live-bootstrap.

 NOTE: this patch CANNOT be applied to a non-live-bootstrap environment.

--- perl-5.32.1/../metaconfig-5.32.1~rc1/dist/U/archname.U	2022-02-26 10:51:45.343097807 +1100
-+++ perl-5.32.1/../metaconfig-5.32.1~rc1/dist/U/archname.U	2022-02-26 10:51:51.742527859 +1100
+--- metaconfig-5.32.1/dist/U/archname.U	2022-02-26 10:51:45.343097807 +1100
+++ metaconfig-5.32.1/dist/U/archname.U	2022-02-26 10:51:51.742527859 +1100
@@ -72,5 +72,5 @@
 rp='What is your architecture name'
 . ./myread
@ -16,8 +16,8 @@ NOTE: this patch CANNOT be applied to a non-live-bootstrap environment.
 -myarchname="$tarch"
 +myarchname="i386"
 
--- perl-5.32.1/../metaconfig-5.32.1~rc1/dist/U/Oldconfig.U	2022-02-27 10:55:04.890396204 +1100
-+++ perl-5.32.1/../metaconfig-5.32.1~rc1/dist/U/Oldconfig.U	2022-02-27 11:00:31.324396204 +1100
+--- metaconfig-5.32.1/dist/U/Oldconfig.U	2022-02-27 10:55:04.890396204 +1100
+++ metaconfig-5.32.1/dist/U/Oldconfig.U	2022-02-27 11:00:31.324396204 +1100
@@ -109,16 +109,13 @@
 ?LINT:extern hostarch libswanted libs
 ?LINT:change hostarch libswanted libs
--- a/steps/perl-5.32.1/sources
+++ b/steps/perl-5.32.1/sources
@ -1,2 +1,2 @@
 http://www.cpan.org/src/5.0/perl-5.32.1.tar.xz 57cc47c735c8300a8ce2fa0643507b44c4ae59012bfdad0121313db639e02309
-http://deb.debian.org/debian/pool/main/p/perl/perl_5.32.1.orig-regen-configure.tar.gz 1d179b41283f12ad83f9758430f6ddc49bdf20db5c396aeae7e51ebb4e4afd29
+git://github.com/Perl/metaconfig~5.32.1 https://github.com/Perl/metaconfig/archive/refs/tags/5.32.1.tar.gz 23abd0d49995229426775c7b1a973e0722faf99bc52e5030d188f0f37973e841
--- a/steps/pkg-config-0.29.2/sources
+++ b/steps/pkg-config-0.29.2/sources
@ -1 +1 @@
-http://distfiles.macports.org/pkgconfig/pkg-config-0.29.2.tar.gz 6fc69c01688c9458a57eb9a1664c9aba372ccda420a02bf4429fe610e7e7d591
+https://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz 6fc69c01688c9458a57eb9a1664c9aba372ccda420a02bf4429fe610e7e7d591
--- a/steps/python-3.11.1/sources
+++ b/steps/python-3.11.1/sources
@ -22,4 +22,3 @@ http://ftp.unicode.org/Public/14.0.0/ucd/Unihan.zip 2ae4519b2b82cd4d15379c17e57b
 http://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP437.TXT 6bad4dabcdf5940227c7d81fab130dcb18a77850b5d79de28b5dc4e047b0aaac
 https://www.ietf.org/rfc/rfc3454.txt eb722fa698fb7e8823b835d9fd263e4cdb8f1c7b0d234edf7f0e3bd2ccbb2c79
 https://github.com/openssl/openssl/releases/download/openssl-3.0.13/openssl-3.0.13.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313
-https://github.com/openssl/openssl/releases/download/openssl-3.0.13/openssl-3.0.13.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313
--- a/steps/sed-4.8/sources
+++ b/steps/sed-4.8/sources
@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/sed/sed-4.8.tar.xz f79b0cfea71b37a8eeec8490db6c5f7ae7719c35587f21edb0617f370eeff633
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-d279bc.tar.gz 12cfa21abf618a274017d6b18e95fc6582519d7c08e2403e5c5772ccdd5b85f4
-https://files.bootstrapping.world/gnulib-d279bc.tar.gz 12cfa21abf618a274017d6b18e95fc6582519d7c08e2403e5c5772ccdd5b85f4
+git://git.savannah.gnu.org/gnulib.git~d279bc _ 12cfa21abf618a274017d6b18e95fc6582519d7c08e2403e5c5772ccdd5b85f4 gnulib-d279bc.tar.gz
--- a/steps/tar-1.34/sources
+++ b/steps/tar-1.34/sources
@ -1,3 +1,2 @@
 http://mirrors.kernel.org/gnu/tar/tar-1.34.tar.xz 63bebd26879c5e1eea4352f0d03c991f966aeb3ddeb3c7445c902568d5411d28
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-30820c.tar.gz df807e694deea2dcba0c43af318394f3e3fcd52658c3b71b61dad0ce0c0cfb77
-https://files.bootstrapping.world/gnulib-30820c.tar.gz df807e694deea2dcba0c43af318394f3e3fcd52658c3b71b61dad0ce0c0cfb77
+git://git.savannah.gnu.org/gnulib.git~30820c _ df807e694deea2dcba0c43af318394f3e3fcd52658c3b71b61dad0ce0c0cfb77 gnulib-30820c.tar.gz
--- a/steps/texinfo-6.7/sources
+++ b/steps/texinfo-6.7/sources
@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/texinfo/texinfo-6.7.tar.xz 988403c1542d15ad044600b909997ba3079b10e03224c61188117f3676b02caa
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-b81ec69.tar.gz 1aeea67b7b3883ebcf2b90bc01f4182d7de073a052dabd3749f20c5aa4ad3e27
-https://files.bootstrapping.world/gnulib-b81ec69.tar.gz 1aeea67b7b3883ebcf2b90bc01f4182d7de073a052dabd3749f20c5aa4ad3e27
+git://git.savannah.gnu.org/gnulib.git~b81ec69 _ 1aeea67b7b3883ebcf2b90bc01f4182d7de073a052dabd3749f20c5aa4ad3e27 gnulib-b81ec69.tar.gz
--- a/steps/xz-5.4.1/sources
+++ b/steps/xz-5.4.1/sources
@ -1 +1 @@
-http://ixpeering.dl.sourceforge.net/project/lzmautils/xz-5.4.1.tar.bz2 dd172acb53867a68012f94c17389401b2f274a1aa5ae8f84cbfb8b7e383ea8d3
+https://github.com/tukaani-project/xz/releases/download/v5.4.1/xz-5.4.1.tar.bz2 dd172acb53867a68012f94c17389401b2f274a1aa5ae8f84cbfb8b7e383ea8d3
--- a/3
+++ b/3
@ -1,3 +0,0 @@
-https://mirrors.kernel.org/gnu/bash/bash-2.05b.tar.gz https://src.fedoraproject.org/repo/pkgs/bash/bash-2.05b.tar.bz2/f3e5428ed52a4f536f571a945d5de95d/bash-2.05b.tar.bz2
-https://mirrors.kernel.org/gnu/bc/bc-1.07.1.tar.gz https://mirrors.kernel.org/slackware/slackware64-15.0/source/ap/bc/bc-1.07.1.tar.xz
-https://mirrors.kernel.org/gnu/ed/ed-1.4.tar.gz https://mirrors.kernel.org/slackware/slackware-13.37/source/a/ed/ed-1.4.tar.xz