fix(payload): switch raw payload metadata fields from u32 to u64

Payload_img_design.md

@@ -43,10 +43,10 @@ The flow is:
 ### Format
 
 - Magic: `LBPAYLD1` (8 bytes)
-- Then: little-endian `u32` file count
+- Then: little-endian `u64` file count
 - Repeated entries:
-  - little-endian `u32` name length
+  - little-endian `u64` name length
-  - little-endian `u32` file size
+  - little-endian `u64` file size
   - file name bytes (no terminator)
   - file bytes
 
@@ -69,21 +69,21 @@ set -e
 out="${1:-external.img}"
 list="${2:-external.list}"
 
-write_u32le() {
+write_u64le() {
   v="$1"
-  printf '%08x' "$v" | sed -E 's/(..)(..)(..)(..)/\4\3\2\1/' | xxd -r -p
+  printf '%016x' "$v" | sed -E 's/(..)(..)(..)(..)(..)(..)(..)(..)/\8\7\6\5\4\3\2\1/' | xxd -r -p
 }
 
 count="$(wc -l < "${list}" | tr -d ' ')"
 : > "${out}"
 printf 'LBPAYLD1' >> "${out}"
-write_u32le "${count}" >> "${out}"
+write_u64le "${count}" >> "${out}"
 
 while read -r name path; do
   [ -n "${name}" ] || continue
   size="$(wc -c < "${path}" | tr -d ' ')"
-  write_u32le "${#name}" >> "${out}"
+  write_u64le "${#name}" >> "${out}"
-  write_u32le "${size}" >> "${out}"
+  write_u64le "${size}" >> "${out}"
   printf '%s' "${name}" >> "${out}"
   cat "${path}" >> "${out}"
 done < "${list}"

README.rst

@@ -71,9 +71,9 @@ Without using Python:
         reach the importer; the rest of the distfiles live in ``external.img``.
 
         * Header magic: ``LBPAYLD1`` (8 bytes).
-        * Then: little-endian ``u32`` file count.
+        * Then: little-endian ``u64`` file count.
-        * Repeated for each file: little-endian ``u32`` name length,
+        * Repeated for each file: little-endian ``u64`` name length,
-          little-endian ``u32`` file size, raw file name bytes, raw file bytes.
+          little-endian ``u64`` file size, raw file name bytes, raw file bytes.
 
       * With ``--repo``, the second disk remains an ext3 distfiles/repo disk.
       * Without ``--external-sources`` and without ``--repo``, no second disk is
@@ -104,21 +104,21 @@ with ``--external-sources`` (and no ``--repo``).
       out="${1:-external.img}"
       list="${2:-external.list}"
 
-      write_u32le() {
+      write_u64le() {
           v="$1"
-          printf '%08x' "$v" | sed -E 's/(..)(..)(..)(..)/\4\3\2\1/' | xxd -r -p
+          printf '%016x' "$v" | sed -E 's/(..)(..)(..)(..)(..)(..)(..)(..)/\8\7\6\5\4\3\2\1/' | xxd -r -p
       }
 
       count="$(wc -l < "${list}" | tr -d ' ')"
       : > "${out}"
       printf 'LBPAYLD1' >> "${out}"
-      write_u32le "${count}" >> "${out}"
+      write_u64le "${count}" >> "${out}"
 
       while read -r name path; do
           [ -n "${name}" ] || continue
           size="$(wc -c < "${path}" | tr -d ' ')"
-          write_u32le "${#name}" >> "${out}"
+          write_u64le "${#name}" >> "${out}"
-          write_u32le "${size}" >> "${out}"
+          write_u64le "${size}" >> "${out}"
           printf '%s' "${name}" >> "${out}"
           cat "${path}" >> "${out}"
       done < "${list}"

lib/generator.py

@@ -153,18 +153,20 @@ class Generator():
         ordered_names = sorted(files_by_name.keys())
         with open(container_path, "wb") as container:
             container.write(self.raw_container_magic)
-            container.write(struct.pack("<I", len(ordered_names)))
+            if len(ordered_names) > 0xFFFFFFFFFFFFFFFF:
+                raise ValueError("Too many files for raw container format.")
+            container.write(struct.pack("<Q", len(ordered_names)))
             for file_name in ordered_names:
                 file_name_bytes = file_name.encode("utf_8")
-                if len(file_name_bytes) > 0xFFFFFFFF:
+                if len(file_name_bytes) > 0xFFFFFFFFFFFFFFFF:
                     raise ValueError(f"Container file name too long: {file_name}")
 
                 src_path = os.path.join(self.distfiles_dir, file_name)
                 file_size = os.path.getsize(src_path)
-                if file_size > 0xFFFFFFFF:
+                if file_size > 0xFFFFFFFFFFFFFFFF:
                     raise ValueError(f"Container file too large for raw container format: {file_name}")
 
-                container.write(struct.pack("<II", len(file_name_bytes), file_size))
+                container.write(struct.pack("<QQ", len(file_name_bytes), file_size))
                 container.write(file_name_bytes)
 
                 with open(src_path, "rb") as src_file:

steps/payload-import-1.0/src/payload-import.c

@@ -14,17 +14,21 @@
 #define COPY_BUFSZ 65536
 #define SYS_MOUNT 21
 
-static unsigned int read_u32le(const unsigned char *buf)
+static unsigned long long read_u64le(const unsigned char *buf)
 {
-	return (unsigned int)buf[0]
+	return (unsigned long long)buf[0]
-		| ((unsigned int)buf[1] << 8)
+		| ((unsigned long long)buf[1] << 8)
-		| ((unsigned int)buf[2] << 16)
+		| ((unsigned long long)buf[2] << 16)
-		| ((unsigned int)buf[3] << 24);
+		| ((unsigned long long)buf[3] << 24)
+		| ((unsigned long long)buf[4] << 32)
+		| ((unsigned long long)buf[5] << 40)
+		| ((unsigned long long)buf[6] << 48)
+		| ((unsigned long long)buf[7] << 56);
 }
 
-static int read_exact(FILE *in, void *buf, unsigned int len)
+static int read_exact(FILE *in, void *buf, size_t len)
 {
-	unsigned int got = 0;
+	size_t got = 0;
 	unsigned char *out = (unsigned char *)buf;
 
 	while (got < len) {
@@ -32,16 +36,16 @@ static int read_exact(FILE *in, void *buf, unsigned int len)
 		if (n == 0) {
 			return -1;
 		}
-		got += (unsigned int)n;
+		got += n;
 	}
 	return 0;
 }
 
-static int copy_exact(FILE *in, FILE *out, unsigned int len,
+static int copy_exact(FILE *in, FILE *out, unsigned long long len,
 	const char *device, const char *name, const char *out_path)
 {
 	unsigned char *buf;
-	unsigned int remaining = len;
+	unsigned long long remaining = len;
 
 	buf = (unsigned char *)malloc(COPY_BUFSZ);
 	if (buf == NULL) {
@@ -50,24 +54,27 @@ static int copy_exact(FILE *in, FILE *out, unsigned int len,
 	}
 
 	while (remaining > 0) {
-		unsigned int chunk = remaining;
+		size_t chunk = (size_t)COPY_BUFSZ;
-		unsigned int copied = len - remaining;
+		unsigned long long copied = len - remaining;
 		size_t nread;
 		size_t written;
-		if (chunk > COPY_BUFSZ) {
+		if (remaining < (unsigned long long)COPY_BUFSZ) {
-			chunk = COPY_BUFSZ;
+			chunk = (size_t)remaining;
 		}
 		nread = fread(buf, 1, chunk, in);
 		if (nread != chunk) {
 			if (feof(in)) {
 				fprintf(stderr,
 					"payload-import: truncated payload while reading %s from %s "
-					"(offset=%u wanted=%u got=%u)\n",
+					"(offset=%llu wanted=%llu got=%llu)\n",
-					name, device, copied, chunk, (unsigned int)nread);
+					name, device,
+					copied,
+					(unsigned long long)chunk,
+					(unsigned long long)nread);
 			} else {
 				fprintf(stderr,
 					"payload-import: read error while reading %s from %s "
-					"(offset=%u): %s\n",
+					"(offset=%llu): %s\n",
 					name, device, copied, strerror(errno));
 			}
 			free(buf);
@@ -77,12 +84,16 @@ static int copy_exact(FILE *in, FILE *out, unsigned int len,
 		if (written != chunk) {
 			fprintf(stderr,
 				"payload-import: write error while writing %s to %s "
-				"(offset=%u wanted=%u wrote=%u): %s\n",
+				"(offset=%llu wanted=%llu wrote=%llu): %s\n",
-				name, out_path, copied, chunk, (unsigned int)written, strerror(errno));
+				name, out_path,
+				copied,
+				(unsigned long long)chunk,
+				(unsigned long long)written,
+				strerror(errno));
 			free(buf);
 			return 1;
 		}
-		remaining -= chunk;
+		remaining -= (unsigned long long)chunk;
 	}
 
 	free(buf);
@@ -133,7 +144,7 @@ static int sys_mount(const char *source, const char *target,
 	const char *fstype, unsigned int flags, const void *data)
 {
 	int ret;
-	// Only for x86 fiwix/linux
+	/* Only for x86 fiwix/linux */
 	__asm__ __volatile__(
 		"int $0x80"
 		: "=a"(ret)
@@ -178,9 +189,9 @@ static int extract_payload(const char *device, const char *dest_dir)
 {
 	FILE *in;
 	char magic[MAGIC_LEN];
-	unsigned char u32buf[4];
+	unsigned char u64buf[8];
-	unsigned int file_count;
+	unsigned long long file_count;
-	unsigned int i;
+	unsigned long long i;
 
 	in = fopen(device, "rb");
 	if (in == NULL) {
@@ -194,15 +205,15 @@ static int extract_payload(const char *device, const char *dest_dir)
 		return 1;
 	}
 
-	if (read_exact(in, u32buf, 4) != 0) {
+	if (read_exact(in, u64buf, 8) != 0) {
 		fclose(in);
 		fputs("payload-import: malformed payload header\n", stderr);
 		return 1;
 	}
-	file_count = read_u32le(u32buf);
+	file_count = read_u64le(u64buf);
-	if (file_count > 200000U) {
+	if (file_count > 200000ULL) {
 		fclose(in);
-		fprintf(stderr, "payload-import: unreasonable file count: %u\n", file_count);
+		fprintf(stderr, "payload-import: unreasonable file count: %llu\n", file_count);
 		return 1;
 	}
 
@@ -212,47 +223,47 @@ static int extract_payload(const char *device, const char *dest_dir)
 		return 1;
 	}
 
-	printf("payload-import: reading %u files from %s\n", file_count, device);
+	printf("payload-import: reading %llu files from %s\n", file_count, device);
 	for (i = 0; i < file_count; ++i) {
-		unsigned int name_len;
+		unsigned long long name_len;
-		unsigned int data_len;
+		unsigned long long data_len;
 		char *name;
 		char out_path[4096];
 		FILE *out;
 
-		if (read_exact(in, u32buf, 4) != 0) {
+		if (read_exact(in, u64buf, 8) != 0) {
 			fclose(in);
 			fputs("payload-import: truncated entry header\n", stderr);
 			return 1;
 		}
-		name_len = read_u32le(u32buf);
+		name_len = read_u64le(u64buf);
-		if (read_exact(in, u32buf, 4) != 0) {
+		if (read_exact(in, u64buf, 8) != 0) {
 			fclose(in);
 			fputs("payload-import: truncated entry size\n", stderr);
 			return 1;
 		}
-		data_len = read_u32le(u32buf);
+		data_len = read_u64le(u64buf);
 
-		if (name_len == 0 || name_len > MAX_NAME_LEN) {
+		if (name_len == 0ULL || name_len > (unsigned long long)MAX_NAME_LEN) {
 			fclose(in);
-			fprintf(stderr, "payload-import: invalid name length %u\n", name_len);
+			fprintf(stderr, "payload-import: invalid name length %llu\n", name_len);
 			return 1;
 		}
 
-		name = (char *)malloc(name_len + 1);
+		name = (char *)malloc((size_t)name_len + 1U);
 		if (name == NULL) {
 			fclose(in);
 			fputs("payload-import: out of memory\n", stderr);
 			return 1;
 		}
 
-		if (read_exact(in, name, name_len) != 0) {
+		if (read_exact(in, name, (size_t)name_len) != 0) {
 			free(name);
 			fclose(in);
 			fputs("payload-import: truncated file name\n", stderr);
 			return 1;
 		}
-		name[name_len] = 0;
+		name[(size_t)name_len] = 0;
 
 		if (!is_valid_name(name)) {
 			fclose(in);