Support mirrors within

DEVEL.md

@@ -84,3 +84,31 @@ libraries that are compiled and installed.
   - Patches are licensed under the license of the project which they are
     patching.
   - All files (excluding files within submodules) must comply with REUSE v3.0.
+
+## `sources` file
+
+The format of the sources file for a HTTP source file is:
+
+```
+<url> <checksum> [filename]
+```
+
+For a Git repository snapshot:
+
+```
+<git url> <URL to HTTP snapshot> <checksum> [filename]
+```
+
+The format of a git url must always be `git://<path to repository>~<reference>`.
+`reference` is, for instance, a commit, tag or branch. Always use
+Git protocol path, except for GitHub, in which `git://` is automagically
+changed to `https://`.
+
+The URL to HTTP snapshot may be `_` (a single underscore) if no HTTP snapshot
+exists. In this case, the filename is compulsory. The checksum is of the Git
+snapshot, generated with `git archive`. See `mirror.sh` for more detailed rules.
+
+Some specific helpful things:
+- prefer `.tar.gz`
+- For GitHub, all snpashots are generated with long commit IDs, so use long
+  commit IDs.

download-distfiles.sh

@@ -9,10 +9,19 @@
 download_source() {
     local distfiles="${1}"
     local url="${2}"
-    local checksum="${3}"
+    shift 2
-    local fname="${4}"
+    if [[ "${url}" == git://* ]]; then
+        url="${1}"
+        shift
+    fi
+    local checksum="${1}"
+    local fname="${2}"
     # Default to basename of url if not given
     fname="${fname:-$(basename "${url}")}"
+    if [ "${fname}" = "_" ]; then
+        echo "ERROR: ${url} must have a filename specified"
+        exit 1
+    fi
 
     local dest_path="${distfiles}/${fname}"
     if ! [ -e "${dest_path}" ]; then

lib/generator.py

@@ -10,9 +10,11 @@ This file contains all code required to generate the boot image for live-bootstr
 
 import hashlib
 import os
+import random
 import shutil
 import tarfile
 import traceback
+
 import requests
 
 # pylint: disable=too-many-instance-attributes
@@ -24,11 +26,13 @@ class Generator():
     git_dir = os.path.join(os.path.dirname(os.path.join(__file__)), '..')
     distfiles_dir = os.path.join(git_dir, 'distfiles')
 
-    def __init__(self, arch, external_sources, early_preseed, repo_path):
+    # pylint: disable=too-many-arguments,too-many-positional-arguments
+    def __init__(self, arch, external_sources, early_preseed, repo_path, mirrors):
         self.arch = arch
         self.early_preseed = early_preseed
         self.external_sources = external_sources
         self.repo_path = repo_path
+        self.mirrors = mirrors
         self.source_manifest = self.get_source_manifest(not self.external_sources)
         self.early_source_manifest = self.get_source_manifest(True)
         self.target_dir = None
@@ -278,8 +282,7 @@ actual:   {readable_hash}\n\
 When in doubt, try deleting the file in question -- it will be downloaded again when running \
 this script the next time")
 
-    @staticmethod
+    def download_file(self, url, directory, file_name, silent=False):
-    def download_file(url, directory, file_name, silent=False):
         """
         Download a single source archive.
         """
@@ -297,14 +300,33 @@ this script the next time")
         if not os.path.isfile(abs_file_name):
             if not silent:
                 print(f"Downloading: {file_name}")
-            response = requests.get(url, allow_redirects=True, stream=True,
+
-                    headers=headers, timeout=20)
+            def do_download(source):
-            if response.status_code == 200:
+                response = requests.get(source, allow_redirects=True, stream=True,
-                with open(abs_file_name, 'wb') as target_file:
+                        headers=headers, timeout=20)
-                    target_file.write(response.raw.read())
+                if response.status_code == 200:
+                    with open(abs_file_name, 'wb') as target_file:
+                        target_file.write(response.raw.read())
+                    return True
+                print(f"Download failed from {option}: {response.status_code} {response.reason}")
+                return False
+
+            done = False
+            if self.mirrors:
+                options = [f"{x}/{file_name}" for x in self.mirrors]
             else:
-                raise requests.HTTPError("Download failed: HTTP " +
+                options = []
-                    str(response.status_code) + " " + response.reason)
+            random.shuffle(options)
+            for option in options:
+                if do_download(option):
+                    done = True
+                    break
+
+            if not done:
+                if url == "_" or not do_download(url):
+                    raise requests.RequestException(f"Unable to download {url} from ",
+                                                    "any mirror or original")
+
         return abs_file_name
 
     def get_packages(self):
@@ -344,6 +366,9 @@ this script the next time")
                         for source in sources.readlines():
                             source = source.strip().split(" ")
 
+                            if source[0].startswith("git://"):
+                                source = source[1:]
+
                             if len(source) > 2:
                                 file_name = source[2]
                             else:

rootfs.py

@@ -50,6 +50,12 @@ def create_configuration_file(args):
             config.write("KERNEL_BOOTSTRAP=False\n")
         config.write(f"BUILD_KERNELS={args.update_checksums or args.build_kernels}\n")
         config.write(f"CONFIGURATOR={args.configurator}\n")
+        if not args.external_sources:
+            if args.mirrors:
+                config.write(f"MIRRORS=\"{" ".join(args.mirrors)}\"\n")
+                config.write(f"MIRRORS_LEN={len(args.mirrors)}\n")
+            else:
+                config.write("MIRRORS_LEN=0\n")
 
 # pylint: disable=too-many-statements,too-many-branches
 def main():
@@ -94,6 +100,9 @@ def main():
     parser.add_argument("--configurator",
                         help="Run the interactive configurator",
                         action="store_true")
+    parser.add_argument("-m", "--mirrors",
+                        help="Mirrors to download distfiles from",
+                        nargs='+')
     parser.add_argument("-r", "--repo",
                         help="Path to prebuilt binary packages", nargs=None)
     parser.add_argument("--early-preseed",
@@ -187,7 +196,8 @@ def main():
     generator = Generator(arch=args.arch,
                           external_sources=args.external_sources,
                           repo_path=args.repo,
-                          early_preseed=args.early_preseed)
+                          early_preseed=args.early_preseed,
+                          mirrors=args.mirrors)
 
     bootstrap(args, generator, target, args.target_size)
 

steps/autogen-5.18.16/sources

@@ -1,6 +1,4 @@
-https://github.com/schierlm/gnu-autogen-bootstrapping/archive/refs/tags/autogen-5.18.16-v1.0.1.tar.gz 953ba180b18acff188a0a8700770c7cf2fc97e1683c7b9699a5a748b542ccdd5
+git://github.com/schierlm/gnu-autogen-bootstrapping~autogen-5.18.16-v1.0.1 https://github.com/schierlm/gnu-autogen-bootstrapping/archive/refs/tags/autogen-5.18.16-v1.0.1.tar.gz 953ba180b18acff188a0a8700770c7cf2fc97e1683c7b9699a5a748b542ccdd5
 https://mirrors.kernel.org/gnu/autogen/rel5.18.16/autogen-5.18.16.tar.xz f8a13466b48faa3ba99fe17a069e71c9ab006d9b1cfabe699f8c60a47d5bb49a
-https://git.savannah.gnu.org/cgit/autogen.git/snapshot/autogen-5.18.16.tar.gz 0c04ab2f7ce13c4a1c06c4abc7dfe75312aad89b8b0a1068e5e563787eb56632
+git://git.savannah.gnu.org/autogen.git~v5.18.16 https://git.savannah.gnu.org/cgit/autogen.git/snapshot/autogen-5.18.16.tar.gz 0c04ab2f7ce13c4a1c06c4abc7dfe75312aad89b8b0a1068e5e563787eb56632
-https://files.bootstrapping.world/autogen-5.18.16.tar.gz 0c04ab2f7ce13c4a1c06c4abc7dfe75312aad89b8b0a1068e5e563787eb56632
+git://git.savannah.gnu.org/gnulib.git~8f4538a5 _ e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab gnulib-8f4538a5.tar.gz
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab
-https://files.bootstrapping.world/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab

steps/bash-2.05b/pass1.kaem

@@ -16,8 +16,8 @@ mkdir build src
 cd build
 
 # Extract
-cp ${DISTFILES}/${pkg}.tar.bz2 ../src/
+cp ${DISTFILES}/${pkg}.tar.gz ../src/
-bzip2 -d -f ../src/${pkg}.tar.bz2
+gzip -d -f ../src/${pkg}.tar.gz
 tar xf ../src/${pkg}.tar
 rm -r ../src/
 cd ${pkg}

steps/bash-2.05b/sources

@@ -1 +1 @@
-https://src.fedoraproject.org/repo/pkgs/bash/bash-2.05b.tar.bz2/f3e5428ed52a4f536f571a945d5de95d/bash-2.05b.tar.bz2 1ce4e5b47a6354531389f0adefb54dee2823227bf6e1e59a31c0e9317a330822
+https://mirrors.kernel.org/gnu/bash/bash-2.05b.tar.gz ba03d412998cc54bd0b0f2d6c32100967d3137098affdc2d32e6e7c11b163fe4

steps/bc-1.07.1/sources

@@ -1 +1 @@
-https://mirrors.kernel.org/slackware/slackware64-15.0/source/ap/bc/bc-1.07.1.tar.xz 95396f88fc719a1bf8bd463809119526fef44a42ab9eb708335c2cb79bc801c6
+https://mirrors.kernel.org/gnu/bc/bc-1.07.1.tar.gz 62adfca89b0a1c0164c2cdca59ca210c1d44c3ffc46daf9931cf4942664cb02a

steps/bison-2.3/sources

@@ -1,3 +1,2 @@
 http://mirrors.kernel.org/gnu/bison/bison-2.3.tar.bz2 b10d7e9e354be72aee4e4911cf19dd27b5c527d4e7200857365b5fcdeea0dffb
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-b28236b.tar.gz 0190f28cb155fedd22bf8558c3e8705eed9eacfb7ae29e7508d025a68eb90899
+git://git.savannah.gnu.org/gnulib.git~b28236b _ 0190f28cb155fedd22bf8558c3e8705eed9eacfb7ae29e7508d025a68eb90899 gnulib-b28236b.tar.gz
-https://files.bootstrapping.world/gnulib-b28236b.tar.gz 0190f28cb155fedd22bf8558c3e8705eed9eacfb7ae29e7508d025a68eb90899

steps/bison-3.4.2/sources

@@ -1,3 +1,2 @@
 http://mirrors.kernel.org/gnu/bison/bison-3.4.2.tar.xz 27d05534699735dc69e86add5b808d6cb35900ad3fd63fa82e3eb644336abfa0
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-672663a.tar.gz 8cced51f89a950472473856f86e88f5daf97a2347756125ccdc8ee907deec570
+git://git.savannah.gnu.org/gnulib.git~672663a _ 8cced51f89a950472473856f86e88f5daf97a2347756125ccdc8ee907deec570 gnulib-672663a.tar.gz
-https://files.bootstrapping.world/gnulib-672663a.tar.gz 8cced51f89a950472473856f86e88f5daf97a2347756125ccdc8ee907deec570

steps/bzip2-1.0.8/pass1.kaem

@@ -15,8 +15,8 @@ mkdir build src
 cd build
 
 # Extract
-cp ${DISTFILES}/${pkg}.tar.xz ../src/
+cp ${DISTFILES}/${pkg}.tar.gz ../src/
-unxz --file ../src/${pkg}.tar.xz --output ../src/${pkg}.tar
+gzip -d -f ../src/${pkg}.tar.gz
 tar xf ../src/${pkg}.tar
 rm -r ../src
 cd ${pkg}

steps/bzip2-1.0.8/sources

@@ -1 +1 @@
-https://mirrors.kernel.org/slackware/slackware-14.0/patches/source/bzip2/bzip2-1.0.8.tar.xz 47fd74b2ff83effad0ddf62074e6fad1f6b4a77a96e121ab421c20a216371a1f
+https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269

steps/coreutils-9.4/sources

@@ -1,7 +1,5 @@
-http://git.savannah.gnu.org/cgit/coreutils.git/snapshot/coreutils-9.4.tar.xz 8fb56810310253300b3d6f84e68dc97eb2d74e1f4f78e05776831d9d82e4f2d7
+git://git.savannah.gnu.org/coreutils.git~v9.4 http://git.savannah.gnu.org/cgit/coreutils.git/snapshot/coreutils-9.4.tar.xz 8fb56810310253300b3d6f84e68dc97eb2d74e1f4f78e05776831d9d82e4f2d7
-https://files.bootstrapping.world/coreutils-9.4.tar.xz 8fb56810310253300b3d6f84e68dc97eb2d74e1f4f78e05776831d9d82e4f2d7
+git://git.savannah.gnu.org/gnulib.git~bb5bb43 _ b8aa1ac1b18c67f081486069e6a7a5564f20431c2313a94c20a46dcfb904be2a gnulib-bb5bb43.tar.gz
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-bb5bb43.tar.gz b8aa1ac1b18c67f081486069e6a7a5564f20431c2313a94c20a46dcfb904be2a
-https://files.bootstrapping.world/gnulib-bb5bb43.tar.gz b8aa1ac1b18c67f081486069e6a7a5564f20431c2313a94c20a46dcfb904be2a
 http://ftp.unicode.org/Public/15.0.0/ucd/UnicodeData.txt 806e9aed65037197f1ec85e12be6e8cd870fc5608b4de0fffd990f689f376a73 UnicodeData-15.0.0.txt
 http://ftp.unicode.org/Public/15.0.0/ucd/PropList.txt e05c0a2811d113dae4abd832884199a3ea8d187ee1b872d8240a788a96540bfd PropList-15.0.0.txt
 http://ftp.unicode.org/Public/15.0.0/ucd/DerivedCoreProperties.txt d367290bc0867e6b484c68370530bdd1a08b6b32404601b8c7accaf83e05628d DerivedCoreProperties-15.0.0.txt

steps/diffutils-3.10/sources

@@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/diffutils/diffutils-3.10.tar.xz 90e5e93cc724e4ebe12ede80df1634063c7a855692685919bfe60b556c9bd09e
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-5d2fe24.tar.gz 72e7bb2d1d75e63d1c46d33b8dd22e8eb60afdba4af3e7251151b5c2a6f00bfb
+git://git.savannah.gnu.org/gnulib.git~5d2fe24 _ 72e7bb2d1d75e63d1c46d33b8dd22e8eb60afdba4af3e7251151b5c2a6f00bfb gnulib-5d2fe24.tar.gz
-https://files.bootstrapping.world/gnulib-5d2fe24.tar.gz 72e7bb2d1d75e63d1c46d33b8dd22e8eb60afdba4af3e7251151b5c2a6f00bfb

steps/ed-1.4/sources

@@ -1 +1 @@
-https://mirrors.kernel.org/slackware/slackware-13.37/source/a/ed/ed-1.4.tar.xz 102c80e6da527c6b8eebd5195cd05fc71808d60735d73e8bb503a5e294475007
+https://mirrors.kernel.org/gnu/ed/ed-1.4.tar.gz db36da85ee1a9d8bafb4b041bd4c8c11becba0c43ec446353b67045de1634fda

steps/findutils-4.2.33/pass1.sh

@@ -3,8 +3,6 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-EXTRA_DISTFILES="gnulib-8e128e.tar.gz"
-
 src_prepare() {
     . ../../import-gnulib.sh
 

steps/findutils-4.2.33/sources

@@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/findutils/findutils-4.2.33.tar.gz 813cd9405aceec5cfecbe96400d01e90ddad7b512d3034487176ce5258ab0f78
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8e128e.tar.gz 0cfbf866bc39c31f25fa0e56af1e56c5e5c92fc1e5d51242ebafef7ea211f3d5
+git://git.savannah.gnu.org/gnulib.git~8e128e _ 0cfbf866bc39c31f25fa0e56af1e56c5e5c92fc1e5d51242ebafef7ea211f3d5 gnulib-8e128e.tar.gz
-https://files.bootstrapping.world/gnulib-8e128e.tar.gz 0cfbf866bc39c31f25fa0e56af1e56c5e5c92fc1e5d51242ebafef7ea211f3d5

steps/gcc-10.4.0/sources

@@ -1 +1 @@
-http://ftp.gnu.org/gnu/gcc/gcc-10.4.0/gcc-10.4.0.tar.xz c9297d5bcd7cb43f3dfc2fed5389e948c9312fd962ef6a4ce455cff963ebe4f1
+http://mirrors.kernel.org/gnu/gcc/gcc-10.4.0/gcc-10.4.0.tar.xz c9297d5bcd7cb43f3dfc2fed5389e948c9312fd962ef6a4ce455cff963ebe4f1

steps/gcc-13.1.0/sources

@@ -1 +1 @@
-https://ftp.gnu.org/gnu/gcc/gcc-13.1.0/gcc-13.1.0.tar.xz 61d684f0aa5e76ac6585ad8898a2427aade8979ed5e7f85492286c4dfc13ee86
+https://mirrors.kernel.org/gnu/gcc/gcc-13.1.0/gcc-13.1.0.tar.xz 61d684f0aa5e76ac6585ad8898a2427aade8979ed5e7f85492286c4dfc13ee86

steps/gettext-0.21/sources

@@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/gettext/gettext-0.21.tar.xz d20fcbb537e02dcf1383197ba05bd0734ef7bf5db06bdb241eb69b7d16b73192
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-7daa86f.tar.gz 2d911c2f2ed97b347d6d360b742abdc98aa626d4f8f847ee682c7cde12e90871
+git://git.savannah.gnu.org/gnulib.git~7daa86f _ 2d911c2f2ed97b347d6d360b742abdc98aa626d4f8f847ee682c7cde12e90871 gnulib-7daa86f.tar.gz
-https://files.bootstrapping.world/gnulib-7daa86f.tar.gz 2d911c2f2ed97b347d6d360b742abdc98aa626d4f8f847ee682c7cde12e90871

steps/grep-3.7/pass1.sh

@@ -2,9 +2,6 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-urls="https://mirrors.kernel.org/gnu/grep/grep-3.7.tar.xz
- http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8f4538a5.tar.gz"
-
 src_prepare() {
     rm configure
     find . -name 'Makefile.in' -delete

steps/grep-3.7/sources

@@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/grep/grep-3.7.tar.xz 5c10da312460aec721984d5d83246d24520ec438dd48d7ab5a05dbc0d6d6823c
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab
+git://git.savannah.gnu.org/gnulib.git~8f4538a5 _ e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab gnulib-8f4538a5.tar.gz
-https://files.bootstrapping.world/gnulib-8f4538a5.tar.gz e207c0bb72093c3a72dde302fcfaa1dbda12a62172d47b73565883a92209ebab

steps/grub-2.06/sources

@@ -1,6 +1,5 @@
 https://mirrors.kernel.org/gnu/grub/grub-2.06.tar.xz b79ea44af91b93d17cd3fe80bdae6ed43770678a9a5ae192ccea803ebb657ee1
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-d271f86.tar.gz 31d69d3d251e39135b5194ddc6f897910d344059f7494d96a739aecbf7ac2b66
+git://git.savannah.gnu.org/gnulib.git~d271f86 _ 31d69d3d251e39135b5194ddc6f897910d344059f7494d96a739aecbf7ac2b66 gnulib-d271f86.tar.gz
-https://files.bootstrapping.world/gnulib-d271f86.tar.gz 31d69d3d251e39135b5194ddc6f897910d344059f7494d96a739aecbf7ac2b66
 http://ftp.unicode.org/Public/9.0.0/ucd/UnicodeData.txt 68dfc414d28257b9b5d6ddbb8b466c768c00ebdf6cbf7784364a9b6cad55ee8f UnicodeData-9.0.0.txt
 http://ftp.unicode.org/Public/9.0.0/ucd/PropList.txt f413ea8dbd3858de72f3148b47dd0586019761357d1481e3b65f3a025bc27f82 PropList-9.0.0.txt
 http://ftp.unicode.org/Public/9.0.0/ucd/DerivedCoreProperties.txt 6662c7e30b572df5d948c092692f52bcc79ab36d49a063a73d6435042db6fb3b DerivedCoreProperties-9.0.0.txt

steps/guile-3.0.9/sources

@@ -1,7 +1,5 @@
 https://mirrors.kernel.org/gnu/guile/guile-3.0.7.tar.xz f57d86c70620271bfceb7a9be0c81744a033f08adc7ceba832c9917ab3e691b7
 https://mirrors.kernel.org/gnu/guile/guile-3.0.9.tar.xz 1a2625ac72b2366e95792f3fe758fd2df775b4044a90a4a9787326e66c0d750d 
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-901694b9.tar.gz f9aad85de1f41d57c9368d304020ffbf354a5e56db1297f022c3d12181134e56
+git://git.savannah.gnu.org/gnulib.git~901694b9 _ f9aad85de1f41d57c9368d304020ffbf354a5e56db1297f022c3d12181134e56 gnulib-901694b9.tar.gz
-https://files.bootstrapping.world/gnulib-901694b9.tar.gz f9aad85de1f41d57c9368d304020ffbf354a5e56db1297f022c3d12181134e56
+git://git.savannah.gnu.org/gnulib.git~356a414e _ fc9973f1a9243fdc4b98d33d7704f3c71bfdc4c2ef96899b8f28cade7290a714 gnulib-356a414e.tar.gz
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-356a414e.tar.gz fc9973f1a9243fdc4b98d33d7704f3c71bfdc4c2ef96899b8f28cade7290a714
+git://github.com/schierlm/guile-psyntax-bootstrapping~guile-3.0.7 https://github.com/schierlm/guile-psyntax-bootstrapping/archive/refs/tags/guile-3.0.7.tar.gz 14cda9c416506dfadf60c14fc623ff01ef99b87564a78d0a29c5d17143c97609
-https://files.bootstrapping.world/gnulib-356a414e.tar.gz fc9973f1a9243fdc4b98d33d7704f3c71bfdc4c2ef96899b8f28cade7290a714
-https://github.com/schierlm/guile-psyntax-bootstrapping/archive/refs/tags/guile-3.0.7.tar.gz 14cda9c416506dfadf60c14fc623ff01ef99b87564a78d0a29c5d17143c97609

steps/gzip-1.13/sources

@@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/gzip/gzip-1.13.tar.xz 7454eb6935db17c6655576c2e1b0fabefd38b4d0936e0f87f48cd062ce91a057
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-5651802.tar.gz 56f1221eb682c3502ee097f583f44673570753cb452346ad4806d94560c3fac9
+git://git.savannah.gnu.org/gnulib.git~5651802 _ 56f1221eb682c3502ee097f583f44673570753cb452346ad4806d94560c3fac9 gnulib-5651802.tar.gz
-https://files.bootstrapping.world/gnulib-5651802.tar.gz 56f1221eb682c3502ee097f583f44673570753cb452346ad4806d94560c3fac9

steps/helpers.sh

@@ -21,8 +21,6 @@ _get_files() {
     prefix="${1}"
     fs=
     if [ -n "$(ls 2>/dev/null)" ]; then
-        # This can be removed once Debian 12 is stable
-        # shellcheck disable=SC2035
         fs=$(echo *)
     fi
     if [ -n "$(ls .[0-z]* 2>/dev/null)" ]; then
@@ -258,18 +256,48 @@ build() {
     unset extract
 }
 
+# An inventive way to randomise with what we know we always have
+randomize() {
+    if command -v shuf >/dev/null 2>&1; then
+        # shellcheck disable=SC2086
+        shuf -e ${1} | tr '\n' ' '
+    else
+        mkdir -p /tmp/random
+        for item in ${1}; do
+            touch --date=@${RANDOM} /tmp/random/"${item//\//~*~*}"
+        done
+        # cannot rely on find existing
+        # shellcheck disable=SC2012
+        ls -1 -t /tmp/random | sed 's:~\*~\*:/:g' | tr '\n' ' '
+        rm -r /tmp/random
+    fi
+}
+
 download_source_line() {
-    url="${1}"
+    if [[ "${1}" == git://* ]]; then
+        shift
+    fi
+    upstream_url="${1}"
     checksum="${2}"
     fname="${3}"
     # Default to basename of url if not given
-    fname="${fname:-$(basename "${url}")}"
+    fname="${fname:-$(basename "${upstream_url}")}"
     if ! [ -e "${fname}" ]; then
-        curl --fail --retry 5 --location "${url}" --output "${fname}" || true
+        for mirror in $(randomize "${MIRRORS}"); do
+            mirror_url="${mirror}/${fname}"
+            echo "${mirror_url}"
+            curl --fail --retry 3 --location "${mirror_url}" --output "${fname}" || true && break
+        done
+        if ! [ -e "${fname}" ] && [ "${upstream_url}" != "_" ]; then
+            curl --fail --retry 3 --location "${upstream_url}" --output "${fname}" || true
+        fi
     fi
 }
 
 check_source_line() {
+    if [[ "${1}" == git://* ]]; then
+        shift
+    fi
     url="${1}"
     checksum="${2}"
     fname="${3}"
@@ -304,6 +332,9 @@ default_src_get() {
 
 # Intelligently extracts a file based upon its filetype.
 extract_file() {
+    if [[ "${1}" == git://* ]]; then
+        shift
+    fi
     f="${3:-$(basename "${1}")}"
     # shellcheck disable=SC2154
     case "${noextract}" in

steps/improve/setup_repo.sh

@@ -6,4 +6,4 @@
 #
 mkdir -p /external/repo
 
-tar -cf - --exclude='/external/repo/*' --exclude='/external/repo-preseeded/*' --exclude='/external/distfiles/*' --exclude='/dev/*' --exclude='/proc/*' --exclude='/sys/*' --exclude='/tmp/*' / | bzip2 --best > /external/repo/base.tar.bz2
+#tar -cf - --exclude='/external/repo/*' --exclude='/external/repo-preseeded/*' --exclude='/external/distfiles/*' --exclude='/dev/*' --exclude='/proc/*' --exclude='/sys/*' --exclude='/tmp/*' / | bzip2 --best > /external/repo/base.tar.bz2

steps/kexec-tools-2.0.22/sources

@@ -1 +1 @@
-https://github.com/horms/kexec-tools/archive/refs/tags/v2.0.22.tar.gz af618de7848142f204b57811f703de3ae7aa3f5bc5d52226db35800fa8fc4dff
+git://github.com/horms/kexec-tools~v2.0.22 https://github.com/horms/kexec-tools/archive/refs/tags/v2.0.22.tar.gz af618de7848142f204b57811f703de3ae7aa3f5bc5d52226db35800fa8fc4dff

steps/libtool-2.4.7/sources

@@ -1,3 +1,2 @@
 http://mirrors.kernel.org/gnu/libtool/libtool-2.4.7.tar.xz 4f7f217f057ce655ff22559ad221a0fd8ef84ad1fc5fcb6990cecc333aa1635d
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-a521820.tar.gz 719b399fe09a8f6ca14ba8c4a9a60ce9f93f4892effb50961ef3d8cd1a33ff65
+git://git.savannah.gnu.org/gnulib.git~a521820 _ 719b399fe09a8f6ca14ba8c4a9a60ce9f93f4892effb50961ef3d8cd1a33ff65 gnulib-a521820.tar.gz
-https://files.bootstrapping.world/gnulib-a521820.tar.gz 719b399fe09a8f6ca14ba8c4a9a60ce9f93f4892effb50961ef3d8cd1a33ff65

steps/libunistring-0.9.10/sources

@@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/libunistring/libunistring-0.9.10.tar.xz eb8fb2c3e4b6e2d336608377050892b54c3c983b646c561836550863003c05d7
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-52a06cb3.tar.gz 009989b81c0bebc5f6550636ed653fbcb237dafc2af5c706f3522087ca571e4d
+git://git.savannah.gnu.org/gnulib.git~52a06cb3 _ 009989b81c0bebc5f6550636ed653fbcb237dafc2af5c706f3522087ca571e4d gnulib-52a06cb3.tar.gz
-https://files.bootstrapping.world/gnulib-52a06cb3.tar.gz 009989b81c0bebc5f6550636ed653fbcb237dafc2af5c706f3522087ca571e4d

steps/m4-1.4.19/sources

@@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/m4/m4-1.4.19.tar.xz 63aede5c6d33b6d9b13511cd0be2cac046f2e70fd0a07aa9573a04a82783af96
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-3639c57.tar.gz 97dfbad67832641bc7f73437617b78abeafb9946723f19cf4c2ceecfc65fa48d
+git://git.savannah.gnu.org/gnulib.git~3639c57 _ 97dfbad67832641bc7f73437617b78abeafb9946723f19cf4c2ceecfc65fa48d gnulib-3639c57.tar.gz
-https://files.bootstrapping.world/gnulib-3639c57.tar.gz 97dfbad67832641bc7f73437617b78abeafb9946723f19cf4c2ceecfc65fa48d

steps/openssl-3.0.13/sources

@@ -1 +1 @@
-http://snapshot.debian.org/archive/debian/20240325T091329Z/pool/main/o/openssl/openssl_3.0.13.orig.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313
+https://github.com/openssl/openssl/releases/download/openssl-3.0.13/openssl-3.0.13.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313

steps/patch-2.7.6/sources

@@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/patch/patch-2.7.6.tar.xz ac610bda97abe0d9f6b7c963255a11dcb196c25e337c61f94e4778d632f1d8fd
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-e017871.tar.gz a285dc300c3d9c25cc06e38827ef40f6073ec3b9b0fcb5bba433f943be92d8d4
+git://git.savannah.gnu.org/gnulib.git~e017871 _ a285dc300c3d9c25cc06e38827ef40f6073ec3b9b0fcb5bba433f943be92d8d4 gnulib-e017871.tar.gz
-https://files.bootstrapping.world/gnulib-e017871.tar.gz a285dc300c3d9c25cc06e38827ef40f6073ec3b9b0fcb5bba433f943be92d8d4

steps/pkg-config-0.29.2/sources

@@ -1 +1 @@
-http://distfiles.macports.org/pkgconfig/pkg-config-0.29.2.tar.gz 6fc69c01688c9458a57eb9a1664c9aba372ccda420a02bf4429fe610e7e7d591
+https://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz 6fc69c01688c9458a57eb9a1664c9aba372ccda420a02bf4429fe610e7e7d591

steps/python-3.11.1/sources

@@ -22,4 +22,3 @@ http://ftp.unicode.org/Public/14.0.0/ucd/Unihan.zip 2ae4519b2b82cd4d15379c17e57b
 http://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP437.TXT 6bad4dabcdf5940227c7d81fab130dcb18a77850b5d79de28b5dc4e047b0aaac
 https://www.ietf.org/rfc/rfc3454.txt eb722fa698fb7e8823b835d9fd263e4cdb8f1c7b0d234edf7f0e3bd2ccbb2c79
 https://github.com/openssl/openssl/releases/download/openssl-3.0.13/openssl-3.0.13.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313
-https://github.com/openssl/openssl/releases/download/openssl-3.0.13/openssl-3.0.13.tar.gz 88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313

steps/sed-4.8/sources

@@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/sed/sed-4.8.tar.xz f79b0cfea71b37a8eeec8490db6c5f7ae7719c35587f21edb0617f370eeff633
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-d279bc.tar.gz 12cfa21abf618a274017d6b18e95fc6582519d7c08e2403e5c5772ccdd5b85f4
+git://git.savannah.gnu.org/gnulib.git~d279bc _ 12cfa21abf618a274017d6b18e95fc6582519d7c08e2403e5c5772ccdd5b85f4 gnulib-d279bc.tar.gz
-https://files.bootstrapping.world/gnulib-d279bc.tar.gz 12cfa21abf618a274017d6b18e95fc6582519d7c08e2403e5c5772ccdd5b85f4

steps/tar-1.34/sources

@@ -1,3 +1,2 @@
 http://mirrors.kernel.org/gnu/tar/tar-1.34.tar.xz 63bebd26879c5e1eea4352f0d03c991f966aeb3ddeb3c7445c902568d5411d28
-http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-30820c.tar.gz df807e694deea2dcba0c43af318394f3e3fcd52658c3b71b61dad0ce0c0cfb77
+git://git.savannah.gnu.org/gnulib.git~30820c _ df807e694deea2dcba0c43af318394f3e3fcd52658c3b71b61dad0ce0c0cfb77 gnulib-30820c.tar.gz
-https://files.bootstrapping.world/gnulib-30820c.tar.gz df807e694deea2dcba0c43af318394f3e3fcd52658c3b71b61dad0ce0c0cfb77

steps/texinfo-6.7/sources

@@ -1,3 +1,2 @@
 https://mirrors.kernel.org/gnu/texinfo/texinfo-6.7.tar.xz 988403c1542d15ad044600b909997ba3079b10e03224c61188117f3676b02caa
-https://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-b81ec69.tar.gz 1aeea67b7b3883ebcf2b90bc01f4182d7de073a052dabd3749f20c5aa4ad3e27
+git://git.savannah.gnu.org/gnulib.git~b81ec69 _ 1aeea67b7b3883ebcf2b90bc01f4182d7de073a052dabd3749f20c5aa4ad3e27 gnulib-b81ec69.tar.gz
-https://files.bootstrapping.world/gnulib-b81ec69.tar.gz 1aeea67b7b3883ebcf2b90bc01f4182d7de073a052dabd3749f20c5aa4ad3e27

steps/xz-5.4.1/sources

@@ -1 +1 @@
-http://ixpeering.dl.sourceforge.net/project/lzmautils/xz-5.4.1.tar.bz2 dd172acb53867a68012f94c17389401b2f274a1aa5ae8f84cbfb8b7e383ea8d3
+https://github.com/tukaani-project/xz/releases/download/v5.4.1/xz-5.4.1.tar.bz2 dd172acb53867a68012f94c17389401b2f274a1aa5ae8f84cbfb8b7e383ea8d3