Make download-distfiles.sh read from steps/manifest

This brings the shell script closer to what get_packages() at
lib/generator.py does.

It allows for downloading only what is specified in the manifest.

.gitignore

@@ -6,5 +6,6 @@
 target/
 kernel
 distfiles/
+mirrorstate/
 __pycache__
 steps/bootstrap.cfg

download-distfiles.sh

@@ -10,10 +10,12 @@ download_source() {
     local distfiles=${1}
     local url=${2}
     shift 2
-    if [[ ${url} == git://* ]]; then
+    case $url in
-        url=${1}
+        'git://'*)
-        shift
+            url=${1}
-    fi
+            shift
+        ;;
+    esac
     local checksum=${1}
     local fname=${2}
     # Default to basename of url if not given
@@ -22,13 +24,12 @@ download_source() {
         echo "ERROR: ${url} must have a filename specified"
         exit 1
     fi
-
     local dest_path=${distfiles}/${fname}
     if ! [ -e "${dest_path}" ]; then
         echo "Downloading ${fname}"
         if [ "${mirrors_len}" -ne 0 ]; then
-            local mirror_ix=$((RANDOM % mirrors_len))
+            local mirror_ix=$(( RANDOM % mirrors_len + 1 ))
-            url=${mirrors[${mirror_ix}]}/${fname}
+            eval "url=\"\${mirror_$mirror_ix}/${fname}\""
         fi
         curl --fail --location "${url}" --output "${dest_path}" || true
     fi
@@ -38,10 +39,12 @@ check_source() {
     local distfiles=${1}
     local url=${2}
     shift 2
-    if [[ ${url} == git://* ]]; then
+    case $url in
-        url=${1}
+        'git://'*)
-        shift
+            url=${1}
-    fi
+            shift
+        ;;
+    esac
     local checksum=${1}
     local fname=${2}
     # Default to basename of url if not given
@@ -51,34 +54,40 @@ check_source() {
     echo "${checksum}  ${dest_path}" | sha256sum -c
 }
 
+walk_manifest_sources () {
+    local action=$1
+    local manifest_entry=
+    local entry=
+
+    while read -r manifest_entry || test -n "$manifest_entry"; do
+        case $manifest_entry in
+            'build:'*)
+                entry="${manifest_entry#'build: '}"
+                entry="${entry%% *}" # remove anything after step name
+                [ -e "./steps/${entry}/sources" ] || continue
+                while read -r line; do
+                    # This is intentional - we want to split out ${line} into separate arguments.
+                    # shellcheck disable=SC2086
+                    $action distfiles ${line}
+                done < "./steps/${entry}/sources"
+                ;;
+        esac
+    done < "./steps/manifest"
+}
+
 set -e
 
-mirrors=( "$@" )
 mirrors_len=$#
+while test $# -gt 0; do
+    eval "mirror_$#=$1"
+    shift
+done
 
 cd "$(dirname "$(readlink -f "$0")")"
 mkdir -p distfiles
 
 # First, try to download anything missing - ignore failing mirrors
-for entry in steps/*; do
+walk_manifest_sources download_source
-    [ -e "${entry}/sources" ] || continue
-
-    # shellcheck disable=SC2162
-    while read line; do
-        # This is intentional - we want to split out ${line} into separate arguments.
-        # shellcheck disable=SC2086
-        download_source distfiles ${line}
-    done < "${entry}/sources"
-done
-
 # Then, check if everything has been obtained at least once
-for entry in steps/*; do
+walk_manifest_sources check_source
-    [ -e "${entry}/sources" ] || continue
 
-    # shellcheck disable=SC2162
-    while read line; do
-        # This is intentional - we want to split out ${line} into separate arguments.
-        # shellcheck disable=SC2086
-        check_source distfiles ${line}
-    done < "${entry}/sources"
-done