From cb08e8038c66661b8bba2ec11a0c2b3d8fb859d2 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 31 Jan 2023 06:10:08 +0000 Subject: [PATCH] Add `tools/pkgrepo.py` to start managing repositories Signed-off-by: Pedro Algarvio --- .pre-commit-config.yaml | 2 + requirements/static/ci/py3.10/tools.txt | 2 + requirements/static/ci/py3.9/tools.txt | 2 + requirements/static/ci/tools.in | 1 + tools/__init__.py | 1 + tools/pkgrepo.py | 282 ++++++++++++++++++++++++ 6 files changed, 290 insertions(+) create mode 100644 tools/pkgrepo.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 71f6bc6b5e6..896c54b0a06 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,6 +19,7 @@ repos: - boto3==1.21.46 - pyyaml==6.0 - jinja2==3.1.2 + - packaging==23.0 - id: tools alias: actionlint name: Lint GitHub Actions Workflows @@ -32,6 +33,7 @@ repos: - boto3==1.21.46 - pyyaml==6.0 - jinja2==3.1.2 + - packaging==23.0 - repo: https://github.com/saltstack/pip-tools-compile-impersonate rev: "4.6" diff --git a/requirements/static/ci/py3.10/tools.txt b/requirements/static/ci/py3.10/tools.txt index a18b1fa9016..0dd56ccebbd 100644 --- a/requirements/static/ci/py3.10/tools.txt +++ b/requirements/static/ci/py3.10/tools.txt @@ -24,6 +24,8 @@ jmespath==1.0.1 # botocore markupsafe==2.1.2 # via jinja2 +packaging==23.0 + # via -r requirements/static/ci/tools.in pygments==2.13.0 # via rich python-dateutil==2.8.2 diff --git a/requirements/static/ci/py3.9/tools.txt b/requirements/static/ci/py3.9/tools.txt index 347617a251e..960a777ba78 100644 --- a/requirements/static/ci/py3.9/tools.txt +++ b/requirements/static/ci/py3.9/tools.txt @@ -24,6 +24,8 @@ jmespath==1.0.1 # botocore markupsafe==2.1.2 # via jinja2 +packaging==23.0 + # via -r requirements/static/ci/tools.in pygments==2.13.0 # via rich python-dateutil==2.8.2 diff --git a/requirements/static/ci/tools.in b/requirements/static/ci/tools.in index 565c1c98b21..e386abcfcd7 100644 --- a/requirements/static/ci/tools.in +++ b/requirements/static/ci/tools.in @@ -3,3 +3,4 @@ attrs boto3 pyyaml jinja2 +packaging diff --git a/tools/__init__.py b/tools/__init__.py index 4180c4cd8ac..d278764a345 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -4,6 +4,7 @@ import tools.changelog import tools.ci import tools.docs import tools.pkg +import tools.pkgrepo import tools.pre_commit import tools.vm diff --git a/tools/pkgrepo.py b/tools/pkgrepo.py new file mode 100644 index 00000000000..f91d23cc52b --- /dev/null +++ b/tools/pkgrepo.py @@ -0,0 +1,282 @@ +""" +These commands are used to build the pacakge repository files. +""" +# pylint: disable=resource-leakage,broad-except +from __future__ import annotations + +import logging +import pathlib +import shutil +import textwrap +from typing import TYPE_CHECKING + +import packaging.version +from ptscripts import Context, command_group + +log = logging.getLogger(__name__) + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent + +# Define the command group +pkg = command_group( + name="pkg-repo", help="Packaging Repository Related Commands", description=__doc__ +) + + +@pkg.command( + name="deb", + arguments={ + "salt_version": { + "help": ( + "The salt version for which to build the repository configuration files. " + "If not passed, it will be discovered by running 'python3 salt/version.py'." + ), + "required": True, + }, + "distro": { + "help": "The debian based distribution to build the repository for", + "choices": ("debian", "ubuntu"), + "required": True, + }, + "distro_version": { + "help": "The distro version.", + "required": True, + }, + "distro_arch": { + "help": "The distribution architecture", + "choices": ("x86_64", "amd64", "aarch64", "arm64"), + }, + "dev_build": { + "help": "Developement repository target", + }, + "repo_path": { + "help": "Path where the repository shall be created.", + "required": True, + }, + "key_id": { + "help": "The GnuPG key ID used to sign.", + "required": True, + }, + "incoming": { + "help": ( + "The path to the directory containing the files that should added to " + "the repository." + ), + "required": True, + }, + }, +) +def debian( + ctx: Context, + salt_version: str = None, + distro: str = None, + distro_version: str = None, + incoming: pathlib.Path = None, + repo_path: pathlib.Path = None, + key_id: str = None, + distro_arch: str = "amd64", + dev_build: bool = False, +): + """ + Create the debian repository. + """ + if TYPE_CHECKING: + assert salt_version is not None + assert distro is not None + assert distro_version is not None + assert incoming is not None + assert repo_path is not None + assert key_id is not None + distro_info = { + "debian": { + "10": { + "label": "deb10ary", + "codename": "buster", + "suitename": "oldstable", + "arm_support": False, + }, + "11": { + "label": "deb11ary", + "codename": "bullseye", + "suitename": "stable", + "arm_support": True, + }, + }, + "ubuntu": { + "18.04": { + "label": "salt_ubuntu1804", + "codename": "bionic", + "arm_support": False, + }, + "20.04": { + "label": "salt_ubuntu2004", + "codename": "focal", + "arm_support": True, + }, + "22.04": { + "label": "salt_ubuntu2204", + "codename": "jammy", + "arm_support": True, + }, + }, + } + display_name = f"{distro.capitalize()} {distro_version}" + if distro_version not in distro_info[distro]: + ctx.error(f"Support for {display_name} is missing.") + ctx.exit(1) + + if distro_arch == "x86_64": + ctx.info(f"The {distro_arch} arch is an alias for 'amd64'. Adjusting.") + distro_arch = "amd64" + + if distro_arch == "aarch64": + ctx.info(f"The {distro_arch} arch is an alias for 'arm64'. Adjusting.") + distro_arch = "arm64" + + distro_details = distro_info[distro][distro_version] + if distro_arch == "arm64" and not distro_details["arm_support"]: + ctx.error(f"There's no arm64 support for {display_name}.") + ctx.exit(1) + + ctx.info("Distribution Details:") + ctx.info(distro_details) + if TYPE_CHECKING: + assert isinstance(distro_details["label"], str) + assert isinstance(distro_details["codename"], str) + assert isinstance(distro_details["suitename"], str) + label: str = distro_details["label"] + codename: str = distro_details["codename"] + + salt_archive_keyring_gpg_file = ( + pathlib.Path("~/salt-archive-keyring.gpg").expanduser().resolve() + ) + if not salt_archive_keyring_gpg_file: + ctx.error(f"The file '{salt_archive_keyring_gpg_file}' does not exist.") + ctx.exit(1) + + ftp_archive_config_suite = "" + if distro == "debian": + suitename: str = distro_details["suitename"] + ftp_archive_config_suite = ( + f"""\n APT::FTPArchive::Release::Suite "{suitename}";\n""" + ) + archive_description = f"SaltProject {display_name} Python 3{'' if dev_build else ' development'} Salt package repo" + ftp_archive_config = f"""\ + APT::FTPArchive::Release::Origin "SaltProject"; + APT::FTPArchive::Release::Label "{label}";{ftp_archive_config_suite} + APT::FTPArchive::Release::Codename "{codename}"; + APT::FTPArchive::Release::Architectures "{distro_arch}"; + APT::FTPArchive::Release::Components "main"; + APT::FTPArchive::Release::Description "{archive_description}"; + APT::FTPArchive::Release::Acquire-By-Hash "yes"; + Dir {{ + ArchiveDir "."; + }}; + BinDirectory "pool" {{ + Packages "dists/{codename}/main/binary-{distro_arch}/Packages"; + Sources "dists/{codename}/main/source/Sources"; + Contents "dists/{codename}/main/Contents-{distro_arch}"; + }} + """ + ctx.info("Creating repository directory structure ...") + create_repo_path = repo_path / distro / distro_version / distro_arch + if dev_build is False: + create_repo_path = create_repo_path / "minor" / salt_version + create_repo_path.mkdir(exist_ok=True, parents=True) + ftp_archive_config_file = create_repo_path / "apt-ftparchive.conf" + ctx.info(f"Writing {ftp_archive_config_file} ...") + ftp_archive_config_file.write_text(textwrap.dedent(ftp_archive_config)) + + ctx.info(f"Copying {salt_archive_keyring_gpg_file} to {create_repo_path} ...") + shutil.copyfile( + salt_archive_keyring_gpg_file, + create_repo_path / salt_archive_keyring_gpg_file.name, + ) + + pool_path = create_repo_path / "pool" + pool_path.mkdir(exist_ok=True) + for fpath in incoming.iterdir(): + dpath = pool_path / fpath.name + ctx.info(f"Copying {fpath} to {dpath} ...") + shutil.copyfile(fpath, dpath) + if fpath.suffix == ".dsc": + ctx.info(f"Running 'debsign' on {dpath} ...") + ctx.run("debsign", "--re-sign", "-k", key_id, str(dpath), interactive=True) + + dists_path = create_repo_path / "dists" + symlink_parent_path = dists_path / codename / "main" + symlink_paths = ( + symlink_parent_path / "by-hash" / "SHA256", + symlink_parent_path / "source" / "by-hash" / "SHA256", + symlink_parent_path / f"binary-{distro_arch}" / "by-hash" / "SHA256", + ) + + for path in symlink_paths: + path.mkdir(exist_ok=True, parents=True) + + cmdline = ["apt-ftparchive", "generate", "apt-ftparchive.conf"] + ctx.info(f"Running '{' '.join(cmdline)}' ...") + ctx.run(*cmdline, cwd=create_repo_path) + + ctx.info("Creating by-hash symlinks ...") + for path in symlink_paths: + for fpath in path.parent.parent.iterdir(): + if not fpath.is_file(): + continue + sha256sum = ctx.run("sha256sum", str(fpath), capture=True) + link = path / sha256sum.stdout.decode().split()[0] + link.symlink_to(f"../../{fpath.name}") + + cmdline = [ + "apt-ftparchive", + "--no-md5", + "--no-sha1", + "--no-sha512", + "release", + "-c", + "apt-ftparchive.conf", + f"dists/{codename}/", + ] + ctx.info(f"Running '{' '.join(cmdline)}' ...") + ret = ctx.run(*cmdline, capture=True, cwd=create_repo_path) + release_file = dists_path / codename / "Release" + ctx.info(f"Writing {release_file} with the output of the previous command...") + release_file.write_bytes(ret.stdout) + + cmdline = [ + "gpg", + "-u", + key_id, + "-o", + f"dists/{codename}/InRelease", + "-a", + "-s", + "--clearsign", + f"dists/{codename}/Release", + ] + ctx.info(f"Running '{' '.join(cmdline)}' ...") + ctx.run(*cmdline, cwd=create_repo_path) + + cmdline = [ + "gpg", + "-u", + key_id, + "-o", + f"dists/{codename}/Release.gpg", + "-a", + "-b", + "-s", + f"dists/{codename}/Release", + ] + + ctx.info(f"Running '{' '.join(cmdline)}' ...") + ctx.run(*cmdline, cwd=create_repo_path) + if dev_build is False: + ctx.info("Creating '' and 'latest' symlinks ...") + major_version = packaging.version.parse(salt_version).major + major_link = create_repo_path.parent.parent / str(major_version) + major_link.symlink_to(f"minor/{salt_version}") + latest_link = create_repo_path.parent.parent / "latest" + latest_link.symlink_to(f"minor/{salt_version}") + + ctx.info("Done")