Create a basic mirroring tool for the new mirror system

.github/workflows/lint.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
       - name: shellcheck
-        run: shellcheck steps/helpers.sh download-distfiles.sh
+        run: shellcheck steps/helpers.sh download-distfiles.sh mirror.sh
 
   reuse:
     name: Lint reuse information

mirror.sh

@@ -0,0 +1,187 @@
+#!/bin/sh
+# This script is intentionally written in POSIX sh to be cross-platform.
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2025 fosslinux <fosslinux@aussies.space>
+
+set -e
+
+# Check arguments, etc
+if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then
+    echo "Usage: $0 <destination> [state]"
+    exit 1
+fi
+
+dest="$1"
+if [ ! -d "${dest}" ]; then
+    echo "${dest} must be a directory"
+    exit 1
+fi
+
+if [ ! -w "${dest}" ]; then
+    echo "you must be able to write to ${dest}"
+    exit 1
+fi
+dest="$(realpath "${dest}")"
+
+state="$2"
+if [ "${state}" = "" ]; then
+    state="${PWD}/mirrorstate"
+fi
+state="$(realpath "${state}")"
+
+#######
+
+# Download a HTTP file
+download_file() {
+    url="${1}"
+    out="${2}"
+    # Download the file
+    continue_arg=""
+    if [ -e "${out}" ]; then
+        continue_arg="-C -"
+    fi
+    # shellcheck disable=SC2086
+    curl -L ${continue_arg} -o "${out}" "${url}"
+}
+
+# Check if a git reference exists in a given repository
+git_ref_exists() {
+    repo="${1}"
+    ref="${2}"
+    # change this to git show-ref once it is sufficiently not-new
+    ( cd "${repo}" || exit && git cat-file -t "${ref}" >/dev/null 2>&1 )
+    return $?
+}
+
+checksum_file() {
+    sha256sum "${1}" | cut -d' ' -f1
+}
+
+do_file() {
+    uri="${1}"
+    echo "${uri}"
+
+    if echo "${uri}" | grep -qE "^https?://"; then
+        # HTTP file
+        checksum="${2}"
+        filename="${3}"
+        if [ "${filename}" = "" ]; then
+            filename="$(basename "${uri}")"
+        fi
+
+        # Check if the file is already downloaded & the checksum is the same
+        dest_file="${dest}/${filename}"
+        if [ -e "${dest_file}" ]; then
+            existing_checksum="$(checksum_file "${dest_file}")"
+            if [ "${checksum}" = "${existing_checksum}" ]; then
+                # There is nothing we need to do here
+                return
+            fi
+        fi
+
+        # Attempt up to 2 times
+        retries=2
+        matching="no"
+        while [ "${retries}" -gt 0 ]; do
+            download_file "${uri}" "${dest}/${filename}"
+            my_checksum="$(checksum_file "${dest_file}")"
+            if [ "${checksum}" = "${my_checksum}" ]; then
+                matching="yes"
+                break
+            fi
+            retries=$((retries - 1))
+            if [ "${retries}" -gt 0 ]; then
+                echo "${uri}: checksum did not match, trying again"
+                rm "${dest}/${filename}"
+            fi
+        done
+
+        if [ "${matching}" = "no" ]; then
+            echo "${uri}: checksums do not match"
+            exit 1
+        fi
+    elif echo "${uri}" | grep -qE "^git://"; then
+        # Creating a tarball from a git repository
+
+        # Very unfortunately, different sites have different rules.
+        uri_path="${uri#git://}"
+        # GitHub does not have git:// protocol support
+        if echo "${uri}" | grep -Eq "^git://github.com"; then
+            uri="https://${uri_path}"
+        fi
+        repo="${uri%~*}"
+        outdir="${state}/git/${repo#*://}"
+        reference="${uri##*~}"
+
+        http_src="${2}"
+        checksum="${3}"
+        tarball="${4:-$(basename "${http_src}")}"
+        if [ "${tarball}" = "_" ]; then
+            echo "${uri}: ERROR! Must have tarball name if no http source."
+            exit 1
+        fi
+        # Heuristic: all tarball filenames start with t
+        suffix="t${tarball#*.t}"
+        tarball="${dest}/${tarball}"
+
+        # Check if tarball already generated + matches checksum
+        checksum="${3}"
+        if [ -e "${tarball}" ]; then
+            existing_checksum="$(checksum_file "${tarball}")"
+            if [ "${existing_checksum}" = "${checksum}" ]; then
+                return
+            fi
+            rm "${tarball}"
+        fi
+
+        # Clone the repository, or update it if needed
+        if [ ! -e "${outdir}" ]; then
+            mkdir -p "$(dirname "${outdir}")"
+            git clone "${repo}" "${outdir}"
+        elif ! git_ref_exists "${outdir}" "${reference}" ; then
+            (
+                cd "${outdir}" || exit
+                git pull
+            )
+        fi
+
+        # Sanity check: the reference we want exists
+        if ! git_ref_exists "${outdir}" "${reference}"; then
+            echo "${reference} not found in ${repo} (${outdir})"
+            exit 1
+        fi
+
+        # Generate the prefix for the tarball
+        prefix_ref="${reference}"
+        # All git repositories we already use remove "v"s from the beginning
+        # of branch/tag names in the tarball prefix
+        if echo "${reference}" | grep -Eq "^v[0-9]"; then
+            prefix_ref="$(echo "${reference}" | sed "s/^v//")"
+        fi
+        prefix="$(basename "${repo}" | sed "s/.git$//")-${prefix_ref}"
+
+        (
+            cd "${outdir}" || exit
+            git config tar.tar.gz.command gzip
+            # -T1 avoids non-determinism due to threading
+            # This may not be correct for forges other than Savannah
+            git config tar.tar.xz.command "xz -T1"
+            git archive "${reference}" -o "${tarball}" --prefix "${prefix}/"
+        )
+
+        my_checksum="$(sha256sum "${tarball}" | cut -d' ' -f1)"
+        if [ "${my_checksum}" != "${checksum}" ]; then
+            echo "${uri}: generated tarball does not match checksum"
+            exit 1
+        fi
+    fi
+}
+
+for src in steps/*/sources; do
+    while read -r line; do
+        # shellcheck disable=SC2086
+        do_file ${line}
+        uri="$(echo "${line}" | cut -d' ' -f1)"
+    done < "${src}"
+done