Support multiple mirrors for each source file

If multiple URLs are entered in a sources listing for the same file,
each will be tried in turn, until either one succeeds, or we fail
having run out of mirrors.

download-distfiles.sh

@@ -15,8 +15,19 @@ download_source() {
     local dest_path="${distfiles}/${fname}"
     if ! [ -e "${dest_path}" ]; then
         echo "Downloading ${fname}"
-        curl --fail --location "${url}" --output "${dest_path}"
+        curl --fail --location "${url}" --output "${dest_path}" || true
     fi
+}
+
+check_source() {
+    local distfiles="${1}"
+    local url="${2}"
+    local checksum="${3}"
+    local fname="${4}"
+    # Default to basename of url if not given
+    fname="${fname:-$(basename "${url}")}"
+
+    local dest_path="${distfiles}/${fname}"
     echo "${checksum}  ${dest_path}" | sha256sum -c
 }
 
@@ -25,6 +36,7 @@ set -e
 cd "$(dirname "$(readlink -f "$0")")"
 mkdir -p distfiles
 
+# First, try to download anything missing - ignore failing mirrors
 for entry in steps/*; do
     [ -e "${entry}/sources" ] || continue
 
@@ -35,3 +47,15 @@ for entry in steps/*; do
         download_source distfiles ${line}
     done < "${entry}/sources"
 done
+
+# Then, check if everything has been obtained at least once
+for entry in steps/*; do
+    [ -e "${entry}/sources" ] || continue
+
+    # shellcheck disable=SC2162
+    while read line; do
+        # This is intentional - we want to split out ${line} into separate arguments.
+        # shellcheck disable=SC2086
+        check_source distfiles ${line}
+    done < "${entry}/sources"
+done