From 693d01dc1b9d3b4d854ae02ccf156f6dfa89a7e9 Mon Sep 17 00:00:00 2001 From: Dor Askayo Date: Fri, 13 Jan 2023 13:15:27 +0200 Subject: [PATCH 1/5] Set git_dir/sys_dir/cache_dir statically This allows accessing their values without requiring a class instance. --- sysa.py | 12 ++++++++---- sysb.py | 7 ++++--- sysc.py | 6 +++--- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/sysa.py b/sysa.py index 0ccdd344..f6ec005c 100755 --- a/sysa.py +++ b/sysa.py @@ -14,26 +14,30 @@ import tarfile from lib.sysgeneral import SysGeneral, stage0_arch_map # pylint: disable=consider-using-with +# pylint: disable=too-many-instance-attributes class SysA(SysGeneral): """ Class responsible for preparing sources for System A. """ - # pylint: disable=too-many-instance-attributes,too-many-arguments + + git_dir = os.path.dirname(os.path.join(__file__)) + sys_dir = os.path.join(git_dir, 'sysa') + cache_dir = os.path.join(sys_dir, 'distfiles') + + # pylint: disable=too-many-arguments def __init__(self, arch, preserve_tmp, external_sources, early_preseed, tmpdir, sysb_dir, sysc_dir): - self.git_dir = os.path.dirname(os.path.join(__file__)) self.arch = arch self.preserve_tmp = preserve_tmp self.early_preseed = early_preseed - self.sys_dir = os.path.join(self.git_dir, 'sysa') if tmpdir is None: self.tmp_dir = os.path.join(self.git_dir, 'tmp') else: self.tmp_dir = os.path.join(tmpdir, 'sysa') self.sysa_dir = os.path.join(self.tmp_dir, 'sysa') self.base_dir = self.sysa_dir - self.cache_dir = os.path.join(self.sys_dir, 'distfiles') + self.sysb_dir = sysb_dir self.sysc_dir = sysc_dir self.external_sources = external_sources diff --git a/sysb.py b/sysb.py index 3f02f48c..d4752f47 100755 --- a/sysb.py +++ b/sysb.py @@ -12,9 +12,10 @@ class SysB(SysGeneral): """ Class responsible for preparing sources for System B. """ + + git_dir = os.path.dirname(os.path.join(__file__)) + sys_dir = os.path.join(git_dir, 'sysb') + def __init__(self, arch, preserve_tmp): - self.git_dir = os.path.dirname(os.path.join(__file__)) self.arch = arch self.preserve_tmp = preserve_tmp - - self.sys_dir = os.path.join(self.git_dir, 'sysb') diff --git a/sysc.py b/sysc.py index 5bbd87c0..b59b2593 100755 --- a/sysc.py +++ b/sysc.py @@ -18,16 +18,16 @@ class SysC(SysGeneral): Class responsible for preparing sources for System C. """ + git_dir = os.path.dirname(os.path.join(__file__)) + sys_dir = os.path.join(git_dir, 'sysc') + cache_dir = os.path.join(sys_dir, 'distfiles') dev_name = None def __init__(self, arch, preserve_tmp, tmpdir, external_sources): - self.git_dir = os.path.dirname(os.path.join(__file__)) self.arch = arch self.preserve_tmp = preserve_tmp self.external_sources = external_sources - self.sys_dir = os.path.join(self.git_dir, 'sysc') - self.cache_dir = os.path.join(self.sys_dir, 'distfiles') if tmpdir is None: self.tmp_dir = os.path.join(self.sys_dir, 'tmp') else: From 38e5dfe35b1b123d0a5bed390aa088df2fb14947 Mon Sep 17 00:00:00 2001 From: Dor Askayo Date: Fri, 13 Jan 2023 15:47:06 +0200 Subject: [PATCH 2/5] Pass destination directory to download_file() --- lib/sysgeneral.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/sysgeneral.py b/lib/sysgeneral.py index 795ad8c4..0241b872 100644 --- a/lib/sysgeneral.py +++ b/lib/sysgeneral.py @@ -3,7 +3,7 @@ This file contains a few functions to be shared by all Sys* classes """ -# SPDX-FileCopyrightText: 2022 Dor Askayo +# SPDX-FileCopyrightText: 2022-2023 Dor Askayo # SPDX-FileCopyrightText: 2021-22 fosslinux # SPDX-FileCopyrightText: 2021 Andrius Štikonas # SPDX-License-Identifier: GPL-3.0-or-later @@ -71,18 +71,18 @@ actual: {readable_hash}\n\ When in doubt, try deleting the file in question -- it will be downloaded again when running \ this script the next time") - def download_file(self, url, file_name=None): + def download_file(self, url, directory, file_name=None): """ Download a single source archive. """ # Automatically determine file name based on URL. if file_name is None: file_name = os.path.basename(url) - abs_file_name = os.path.join(self.cache_dir, file_name) + abs_file_name = os.path.join(directory, file_name) - # Create a cache directory for downloaded sources - if not os.path.isdir(self.cache_dir): - os.mkdir(self.cache_dir) + # Create a directory for downloaded file + if not os.path.isdir(directory): + os.mkdir(directory) # Actually download the file headers = { @@ -111,9 +111,9 @@ this script the next time") for line in sources.readlines(): line = line.strip().split(" ") if len(line) > 2: - path = self.download_file(line[0], line[2]) + path = self.download_file(line[0], self.cache_dir, line[2]) else: - path = self.download_file(line[0]) + path = self.download_file(line[0], self.cache_dir) self.check_file(path, line[1]) def make_initramfs(self): From 425beee8dda68683fa2c34830a1e1f0ecf1af953 Mon Sep 17 00:00:00 2001 From: Dor Askayo Date: Fri, 13 Jan 2023 19:38:28 +0200 Subject: [PATCH 3/5] Always pass file name to download_file() Move the file name decision to get_packages() when a file name is not specified in the package source file. --- lib/sysgeneral.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/sysgeneral.py b/lib/sysgeneral.py index 0241b872..fdbafd06 100644 --- a/lib/sysgeneral.py +++ b/lib/sysgeneral.py @@ -71,13 +71,10 @@ actual: {readable_hash}\n\ When in doubt, try deleting the file in question -- it will be downloaded again when running \ this script the next time") - def download_file(self, url, directory, file_name=None): + def download_file(self, url, directory, file_name): """ Download a single source archive. """ - # Automatically determine file name based on URL. - if file_name is None: - file_name = os.path.basename(url) abs_file_name = os.path.join(directory, file_name) # Create a directory for downloaded file @@ -110,10 +107,14 @@ this script the next time") with open(sourcef, "r", encoding="utf_8") as sources: for line in sources.readlines(): line = line.strip().split(" ") + if len(line) > 2: - path = self.download_file(line[0], self.cache_dir, line[2]) + file_name = line[2] else: - path = self.download_file(line[0], self.cache_dir) + # Automatically determine file name based on URL. + file_name = os.path.basename(line[0]) + + path = self.download_file(line[0], self.cache_dir, file_name) self.check_file(path, line[1]) def make_initramfs(self): From a1c8c0312c25e8dbd93951b4298063a8d1a453bb Mon Sep 17 00:00:00 2001 From: Dor Askayo Date: Fri, 13 Jan 2023 15:55:39 +0200 Subject: [PATCH 4/5] Download source files using a source manifest --- lib/sysgeneral.py | 29 ++++++++++++++++++++++------- sysa.py | 5 +++-- sysc.py | 6 ++++-- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/lib/sysgeneral.py b/lib/sysgeneral.py index fdbafd06..e1d2efa4 100644 --- a/lib/sysgeneral.py +++ b/lib/sysgeneral.py @@ -96,14 +96,28 @@ this script the next time") raise Exception("Download failed.") return abs_file_name - def get_packages(self): + def get_packages(self, source_manifest): """Prepare remaining sources""" + for line in source_manifest.split("\n"): + line = line.strip().split(" ") + + path = self.download_file(line[2], line[1], line[3]) + self.check_file(path, line[0]) + + @classmethod + def get_source_manifest(cls): + """ + Generage a source manifest for the system. + """ + manifest_lines = [] + directory = os.path.relpath(cls.cache_dir, cls.git_dir) + # Find all source files - for file in os.listdir(self.sys_dir): - if os.path.isdir(os.path.join(self.sys_dir, file)): - sourcef = os.path.join(self.sys_dir, file, "sources") + for file in os.listdir(cls.sys_dir): + if os.path.isdir(os.path.join(cls.sys_dir, file)): + sourcef = os.path.join(cls.sys_dir, file, "sources") if os.path.exists(sourcef): - # Download sources in the source file + # Read sources from the source file with open(sourcef, "r", encoding="utf_8") as sources: for line in sources.readlines(): line = line.strip().split(" ") @@ -114,8 +128,9 @@ this script the next time") # Automatically determine file name based on URL. file_name = os.path.basename(line[0]) - path = self.download_file(line[0], self.cache_dir, file_name) - self.check_file(path, line[1]) + manifest_lines.append(f"{line[1]} {directory} {line[0]} {file_name}") + + return "\n".join(manifest_lines) def make_initramfs(self): """Package binary bootstrap seeds and sources into initramfs.""" diff --git a/sysa.py b/sysa.py index f6ec005c..dd709c9e 100755 --- a/sysa.py +++ b/sysa.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """System A""" # SPDX-License-Identifier: GPL-3.0-or-later -# SPDX-FileCopyrightText: 2022 Dor Askayo +# SPDX-FileCopyrightText: 2022-2023 Dor Askayo # SPDX-FileCopyrightText: 2021 Andrius Štikonas # SPDX-FileCopyrightText: 2021 Melg Eight # SPDX-FileCopyrightText: 2021-22 fosslinux @@ -78,7 +78,8 @@ class SysA(SysGeneral): def sysa(self): """Copy in sysa files for sysa.""" - self.get_packages() + source_manifest = self.get_source_manifest() + self.get_packages(source_manifest) shutil.copytree(self.sys_dir, os.path.join(self.tmp_dir, 'sysa'), ignore=shutil.ignore_patterns('tmp')) diff --git a/sysc.py b/sysc.py index b59b2593..0851f25f 100755 --- a/sysc.py +++ b/sysc.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """System C""" # SPDX-License-Identifier: GPL-3.0-or-later -# SPDX-FileCopyrightText: 2022 Dor Askayo +# SPDX-FileCopyrightText: 2022-2023 Dor Askayo # SPDX-FileCopyrightText: 2021-22 fosslinux # SPDX-FileCopyrightText: 2021 Andrius Štikonas @@ -70,7 +70,9 @@ class SysC(SysGeneral): rootfs_dir = self.tmp_dir if self.external_sources: - self.get_packages() + source_manifest = self.get_source_manifest() + self.get_packages(source_manifest) + copytree(self.cache_dir, os.path.join(rootfs_dir, "distfiles")) # Unmount tmp/mnt if it was mounted From 9d1e5b64beeb74357fb3305fb14e3884fa26ec84 Mon Sep 17 00:00:00 2001 From: Dor Askayo Date: Fri, 13 Jan 2023 15:57:22 +0200 Subject: [PATCH 5/5] Add a script to generate source manifests The source manifest can be used by external tools to download source files externally before initiating the bootstrap process. The script prints the source manifest to stdout. --- source-manifest.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100755 source-manifest.py diff --git a/source-manifest.py b/source-manifest.py new file mode 100755 index 00000000..88a57a9c --- /dev/null +++ b/source-manifest.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +""" +A helper application used to get a list of source files required +for the bootstrapping process. +""" + +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2023 Dor Askayo + +import argparse + +from sysa import SysA +from sysc import SysC + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument("-s", "--system", + help="Generate source manifest for the specified systems", + choices=["sysa", "sysc"], + nargs="+", + action="extend", + required=True) + + args = parser.parse_args() + + if "sysa" in args.system: + print(SysA.get_source_manifest()) + + if "sysc" in args.system: + print(SysC.get_source_manifest()) + +if __name__ == "__main__": + main()