diff --git a/Payload_img_design.md b/Payload_img_design.md index 0060abc5..4f223ecc 100644 --- a/Payload_img_design.md +++ b/Payload_img_design.md @@ -43,10 +43,10 @@ The flow is: ### Format - Magic: `LBPAYLD1` (8 bytes) -- Then: little-endian `u32` file count +- Then: little-endian `u64` file count - Repeated entries: - - little-endian `u32` name length - - little-endian `u32` file size + - little-endian `u64` name length + - little-endian `u64` file size - file name bytes (no terminator) - file bytes @@ -69,21 +69,21 @@ set -e out="${1:-external.img}" list="${2:-external.list}" -write_u32le() { +write_u64le() { v="$1" - printf '%08x' "$v" | sed -E 's/(..)(..)(..)(..)/\4\3\2\1/' | xxd -r -p + printf '%016x' "$v" | sed -E 's/(..)(..)(..)(..)(..)(..)(..)(..)/\8\7\6\5\4\3\2\1/' | xxd -r -p } count="$(wc -l < "${list}" | tr -d ' ')" : > "${out}" printf 'LBPAYLD1' >> "${out}" -write_u32le "${count}" >> "${out}" +write_u64le "${count}" >> "${out}" while read -r name path; do [ -n "${name}" ] || continue size="$(wc -c < "${path}" | tr -d ' ')" - write_u32le "${#name}" >> "${out}" - write_u32le "${size}" >> "${out}" + write_u64le "${#name}" >> "${out}" + write_u64le "${size}" >> "${out}" printf '%s' "${name}" >> "${out}" cat "${path}" >> "${out}" done < "${list}" diff --git a/README.rst b/README.rst index 1eb627e8..210506db 100644 --- a/README.rst +++ b/README.rst @@ -71,9 +71,9 @@ Without using Python: reach the importer; the rest of the distfiles live in ``external.img``. * Header magic: ``LBPAYLD1`` (8 bytes). - * Then: little-endian ``u32`` file count. - * Repeated for each file: little-endian ``u32`` name length, - little-endian ``u32`` file size, raw file name bytes, raw file bytes. + * Then: little-endian ``u64`` file count. + * Repeated for each file: little-endian ``u64`` name length, + little-endian ``u64`` file size, raw file name bytes, raw file bytes. * With ``--repo``, the second disk remains an ext3 distfiles/repo disk. * Without ``--external-sources`` and without ``--repo``, no second disk is @@ -104,21 +104,21 @@ with ``--external-sources`` (and no ``--repo``). out="${1:-external.img}" list="${2:-external.list}" - write_u32le() { + write_u64le() { v="$1" - printf '%08x' "$v" | sed -E 's/(..)(..)(..)(..)/\4\3\2\1/' | xxd -r -p + printf '%016x' "$v" | sed -E 's/(..)(..)(..)(..)(..)(..)(..)(..)/\8\7\6\5\4\3\2\1/' | xxd -r -p } count="$(wc -l < "${list}" | tr -d ' ')" : > "${out}" printf 'LBPAYLD1' >> "${out}" - write_u32le "${count}" >> "${out}" + write_u64le "${count}" >> "${out}" while read -r name path; do [ -n "${name}" ] || continue size="$(wc -c < "${path}" | tr -d ' ')" - write_u32le "${#name}" >> "${out}" - write_u32le "${size}" >> "${out}" + write_u64le "${#name}" >> "${out}" + write_u64le "${size}" >> "${out}" printf '%s' "${name}" >> "${out}" cat "${path}" >> "${out}" done < "${list}" diff --git a/lib/generator.py b/lib/generator.py index 3d7e3c6a..c8487d0f 100755 --- a/lib/generator.py +++ b/lib/generator.py @@ -153,18 +153,20 @@ class Generator(): ordered_names = sorted(files_by_name.keys()) with open(container_path, "wb") as container: container.write(self.raw_container_magic) - container.write(struct.pack(" 0xFFFFFFFFFFFFFFFF: + raise ValueError("Too many files for raw container format.") + container.write(struct.pack(" 0xFFFFFFFF: + if len(file_name_bytes) > 0xFFFFFFFFFFFFFFFF: raise ValueError(f"Container file name too long: {file_name}") src_path = os.path.join(self.distfiles_dir, file_name) file_size = os.path.getsize(src_path) - if file_size > 0xFFFFFFFF: + if file_size > 0xFFFFFFFFFFFFFFFF: raise ValueError(f"Container file too large for raw container format: {file_name}") - container.write(struct.pack(" 0) { - unsigned int chunk = remaining; - unsigned int copied = len - remaining; + size_t chunk = (size_t)COPY_BUFSZ; + unsigned long long copied = len - remaining; size_t nread; size_t written; - if (chunk > COPY_BUFSZ) { - chunk = COPY_BUFSZ; + if (remaining < (unsigned long long)COPY_BUFSZ) { + chunk = (size_t)remaining; } nread = fread(buf, 1, chunk, in); if (nread != chunk) { if (feof(in)) { fprintf(stderr, "payload-import: truncated payload while reading %s from %s " - "(offset=%u wanted=%u got=%u)\n", - name, device, copied, chunk, (unsigned int)nread); + "(offset=%llu wanted=%llu got=%llu)\n", + name, device, + copied, + (unsigned long long)chunk, + (unsigned long long)nread); } else { fprintf(stderr, "payload-import: read error while reading %s from %s " - "(offset=%u): %s\n", + "(offset=%llu): %s\n", name, device, copied, strerror(errno)); } free(buf); @@ -77,12 +84,16 @@ static int copy_exact(FILE *in, FILE *out, unsigned int len, if (written != chunk) { fprintf(stderr, "payload-import: write error while writing %s to %s " - "(offset=%u wanted=%u wrote=%u): %s\n", - name, out_path, copied, chunk, (unsigned int)written, strerror(errno)); + "(offset=%llu wanted=%llu wrote=%llu): %s\n", + name, out_path, + copied, + (unsigned long long)chunk, + (unsigned long long)written, + strerror(errno)); free(buf); return 1; } - remaining -= chunk; + remaining -= (unsigned long long)chunk; } free(buf); @@ -133,7 +144,7 @@ static int sys_mount(const char *source, const char *target, const char *fstype, unsigned int flags, const void *data) { int ret; - // Only for x86 fiwix/linux + /* Only for x86 fiwix/linux */ __asm__ __volatile__( "int $0x80" : "=a"(ret) @@ -178,9 +189,9 @@ static int extract_payload(const char *device, const char *dest_dir) { FILE *in; char magic[MAGIC_LEN]; - unsigned char u32buf[4]; - unsigned int file_count; - unsigned int i; + unsigned char u64buf[8]; + unsigned long long file_count; + unsigned long long i; in = fopen(device, "rb"); if (in == NULL) { @@ -194,15 +205,15 @@ static int extract_payload(const char *device, const char *dest_dir) return 1; } - if (read_exact(in, u32buf, 4) != 0) { + if (read_exact(in, u64buf, 8) != 0) { fclose(in); fputs("payload-import: malformed payload header\n", stderr); return 1; } - file_count = read_u32le(u32buf); - if (file_count > 200000U) { + file_count = read_u64le(u64buf); + if (file_count > 200000ULL) { fclose(in); - fprintf(stderr, "payload-import: unreasonable file count: %u\n", file_count); + fprintf(stderr, "payload-import: unreasonable file count: %llu\n", file_count); return 1; } @@ -212,47 +223,47 @@ static int extract_payload(const char *device, const char *dest_dir) return 1; } - printf("payload-import: reading %u files from %s\n", file_count, device); + printf("payload-import: reading %llu files from %s\n", file_count, device); for (i = 0; i < file_count; ++i) { - unsigned int name_len; - unsigned int data_len; + unsigned long long name_len; + unsigned long long data_len; char *name; char out_path[4096]; FILE *out; - if (read_exact(in, u32buf, 4) != 0) { + if (read_exact(in, u64buf, 8) != 0) { fclose(in); fputs("payload-import: truncated entry header\n", stderr); return 1; } - name_len = read_u32le(u32buf); - if (read_exact(in, u32buf, 4) != 0) { + name_len = read_u64le(u64buf); + if (read_exact(in, u64buf, 8) != 0) { fclose(in); fputs("payload-import: truncated entry size\n", stderr); return 1; } - data_len = read_u32le(u32buf); + data_len = read_u64le(u64buf); - if (name_len == 0 || name_len > MAX_NAME_LEN) { + if (name_len == 0ULL || name_len > (unsigned long long)MAX_NAME_LEN) { fclose(in); - fprintf(stderr, "payload-import: invalid name length %u\n", name_len); + fprintf(stderr, "payload-import: invalid name length %llu\n", name_len); return 1; } - name = (char *)malloc(name_len + 1); + name = (char *)malloc((size_t)name_len + 1U); if (name == NULL) { fclose(in); fputs("payload-import: out of memory\n", stderr); return 1; } - if (read_exact(in, name, name_len) != 0) { + if (read_exact(in, name, (size_t)name_len) != 0) { free(name); fclose(in); fputs("payload-import: truncated file name\n", stderr); return 1; } - name[name_len] = 0; + name[(size_t)name_len] = 0; if (!is_valid_name(name)) { fclose(in);