From 4c8cc6062904bd0fab29ef4272eef6c20c71d392 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 9 Feb 2023 06:18:37 +0000 Subject: [PATCH] Add `tools/utils.py` and move some common functionality there. Signed-off-by: Pedro Algarvio --- tools/changelog.py | 7 ++- tools/ci.py | 16 ++++--- tools/docs.py | 6 +-- tools/pkg.py | 30 +++++++------ tools/pkgrepo.py | 107 +++++++------------------------------------- tools/pre_commit.py | 12 ++--- tools/utils.py | 88 ++++++++++++++++++++++++++++++++++++ tools/vm.py | 17 +++---- 8 files changed, 151 insertions(+), 132 deletions(-) create mode 100644 tools/utils.py diff --git a/tools/changelog.py b/tools/changelog.py index 1df867bd61c..71f6a209b10 100644 --- a/tools/changelog.py +++ b/tools/changelog.py @@ -7,16 +7,15 @@ from __future__ import annotations import datetime import logging import os -import pathlib import subprocess import sys import textwrap from ptscripts import Context, command_group -log = logging.getLogger(__name__) +import tools.utils -REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +log = logging.getLogger(__name__) # Define the command group changelog = command_group( @@ -25,7 +24,7 @@ changelog = command_group( description=__doc__, venv_config={ "requirements_files": [ - REPO_ROOT + tools.utils.REPO_ROOT / "requirements" / "static" / "ci" diff --git a/tools/ci.py b/tools/ci.py index 1c8f856d3f8..8c2c1f60e11 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -13,9 +13,9 @@ from typing import TYPE_CHECKING from ptscripts import Context, command_group -log = logging.getLogger(__name__) +import tools.utils -REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +log = logging.getLogger(__name__) # Define the command group ci = command_group(name="ci", help="CI Related Commands", description=__doc__) @@ -107,7 +107,11 @@ def process_changed_files(ctx: Context, event_name: str, changed_files: pathlib. if not entry: loaded_data.remove(entry) try: - entry = REPO_ROOT.joinpath(entry).resolve().relative_to(REPO_ROOT) + entry = ( + tools.utils.REPO_ROOT.joinpath(entry) + .resolve() + .relative_to(tools.utils.REPO_ROOT) + ) except ValueError: ctx.error( f"While processing the changed files key {key!r}, the " @@ -417,10 +421,12 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path): wfh.write("\n\n") testrun = {"type": "full"} else: - testrun_changed_files_path = REPO_ROOT / "testrun-changed-files.txt" + testrun_changed_files_path = tools.utils.REPO_ROOT / "testrun-changed-files.txt" testrun = { "type": "changed", - "from-filenames": str(testrun_changed_files_path.relative_to(REPO_ROOT)), + "from-filenames": str( + testrun_changed_files_path.relative_to(tools.utils.REPO_ROOT) + ), } ctx.info(f"Writing {testrun_changed_files_path.name} ...") selected_changed_files = [] diff --git a/tools/docs.py b/tools/docs.py index d3b619eb12d..af380efc52b 100644 --- a/tools/docs.py +++ b/tools/docs.py @@ -12,9 +12,9 @@ import sys from ptscripts import Context, command_group -log = logging.getLogger(__name__) +import tools.utils -REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +log = logging.getLogger(__name__) # Define the command group docs = command_group( @@ -23,7 +23,7 @@ docs = command_group( description=__doc__, venv_config={ "requirements_files": [ - REPO_ROOT + tools.utils.REPO_ROOT / "requirements" / "static" / "ci" diff --git a/tools/pkg.py b/tools/pkg.py index 23029eb66c6..7ff2f286849 100644 --- a/tools/pkg.py +++ b/tools/pkg.py @@ -19,9 +19,9 @@ import tempfile import yaml from ptscripts import Context, command_group -log = logging.getLogger(__name__) +import tools.utils -REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +log = logging.getLogger(__name__) # Define the command group pkg = command_group(name="pkg", help="Packaging Related Commands", description=__doc__) @@ -112,14 +112,14 @@ def set_salt_version( """ Write the Salt version to 'salt/_version.txt' """ - salt_version_file = REPO_ROOT / "salt" / "_version.txt" + salt_version_file = tools.utils.REPO_ROOT / "salt" / "_version.txt" if salt_version_file.exists(): if not overwrite: ctx.error("The 'salt/_version.txt' file already exists") ctx.exit(1) salt_version_file.unlink() if salt_version is None: - if not REPO_ROOT.joinpath(".git").exists(): + if not tools.utils.REPO_ROOT.joinpath(".git").exists(): ctx.error( "Apparently not running from a Salt repository checkout. " "Unable to discover the Salt version." @@ -133,7 +133,7 @@ def set_salt_version( ctx.info(f"Validating and normalizing the salt version {salt_version!r}...") with ctx.virtualenv( name="set-salt-version", - requirements_files=[REPO_ROOT / "requirements" / "base.txt"], + requirements_files=[tools.utils.REPO_ROOT / "requirements" / "base.txt"], ) as venv: code = f""" import sys @@ -153,14 +153,14 @@ def set_salt_version( ctx.exit(ctx.returncode) salt_version = ret.stdout.strip().decode() - if not REPO_ROOT.joinpath("salt").is_dir(): + if not tools.utils.REPO_ROOT.joinpath("salt").is_dir(): ctx.error( "The path 'salt/' is not a directory. Unable to write 'salt/_version.txt'" ) ctx.exit(1) try: - REPO_ROOT.joinpath("salt/_version.txt").write_text(salt_version) + tools.utils.REPO_ROOT.joinpath("salt/_version.txt").write_text(salt_version) except Exception as exc: ctx.error(f"Unable to write 'salt/_version.txt': {exc}") ctx.exit(1) @@ -211,7 +211,9 @@ def pre_archive_cleanup(ctx: Context, cleanup_path: str, pkg: bool = False): When running on Windows and macOS, some additional cleanup is also done. """ - with open(str(REPO_ROOT / "pkg" / "common" / "env-cleanup-rules.yml")) as rfh: + with open( + str(tools.utils.REPO_ROOT / "pkg" / "common" / "env-cleanup-rules.yml") + ) as rfh: patterns = yaml.safe_load(rfh.read()) if pkg: @@ -317,7 +319,7 @@ def generate_hashes(ctx: Context, files: list[pathlib.Path]): name="source-tarball", venv_config={ "requirements_files": [ - REPO_ROOT / "requirements" / "build.txt", + tools.utils.REPO_ROOT / "requirements" / "build.txt", ] }, ) @@ -342,20 +344,20 @@ def source_tarball(ctx: Context): "-m", "build", "--sdist", - str(REPO_ROOT), + str(tools.utils.REPO_ROOT), env=env, check=True, ) # Recreate sdist to be reproducible recompress = Recompress(timestamp) - for targz in REPO_ROOT.joinpath("dist").glob("*.tar.gz"): - ctx.info(f"Re-compressing {targz.relative_to(REPO_ROOT)} ...") + for targz in tools.utils.REPO_ROOT.joinpath("dist").glob("*.tar.gz"): + ctx.info(f"Re-compressing {targz.relative_to(tools.utils.REPO_ROOT)} ...") recompress.recompress(targz) sha256sum = shutil.which("sha256sum") if sha256sum: packages = [ - str(pkg.relative_to(REPO_ROOT)) - for pkg in REPO_ROOT.joinpath("dist").iterdir() + str(pkg.relative_to(tools.utils.REPO_ROOT)) + for pkg in tools.utils.REPO_ROOT.joinpath("dist").iterdir() ] ctx.run("sha256sum", *packages) ctx.run("python3", "-m", "twine", "check", "dist/*", check=True) diff --git a/tools/pkgrepo.py b/tools/pkgrepo.py index 7b8e524ed3c..89182da1718 100644 --- a/tools/pkgrepo.py +++ b/tools/pkgrepo.py @@ -19,19 +19,11 @@ import packaging.version from ptscripts import Context, command_group import tools.pkg +import tools.utils try: import boto3 from botocore.exceptions import ClientError - from rich.progress import ( - BarColumn, - Column, - DownloadColumn, - Progress, - TextColumn, - TimeRemainingColumn, - TransferSpeedColumn, - ) except ImportError: print( "\nPlease run 'python -m pip install -r " @@ -43,12 +35,6 @@ except ImportError: log = logging.getLogger(__name__) -REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent -GPG_KEY_FILENAME = "SALT-PROJECT-GPG-PUBKEY-2023" -NIGHTLY_BUCKET_NAME = "salt-project-prod-salt-artifacts-nightly" -STAGING_BUCKET_NAME = "salt-project-prod-salt-artifacts-staging" -RELEASE_BUCKET_NAME = "salt-project-prod-salt-artifacts-release" - # Define the command group repo = command_group( name="repo", @@ -227,7 +213,7 @@ def debian( ftp_archive_config_file.write_text(textwrap.dedent(ftp_archive_config)) # Export the GPG key in use - _export_gpg_key(ctx, key_id, repo_path, create_repo_path) + tools.utils.export_gpg_key(ctx, key_id, repo_path, create_repo_path) pool_path = create_repo_path / "pool" pool_path.mkdir(exist_ok=True) @@ -415,7 +401,7 @@ def rpm( ) # Export the GPG key in use - _export_gpg_key(ctx, key_id, repo_path, create_repo_path) + tools.utils.export_gpg_key(ctx, key_id, repo_path, create_repo_path) for fpath in incoming.iterdir(): if ".src" in fpath.suffixes: @@ -775,9 +761,9 @@ def _create_onedir_based_repo( repo_json_path = create_repo_path.parent / "repo.json" if nightly_build: - bucket_name = NIGHTLY_BUCKET_NAME + bucket_name = tools.utils.NIGHTLY_BUCKET_NAME else: - bucket_name = STAGING_BUCKET_NAME + bucket_name = tools.utils.STAGING_BUCKET_NAME release_json = {} @@ -827,24 +813,10 @@ def _create_onedir_based_repo( for fpath in create_repo_path.iterdir(): if fpath.suffix in pkg_suffixes: continue - ctx.info(f"GPG Signing '{fpath.relative_to(repo_path)}' ...") - signature_fpath = fpath.parent / f"{fpath.name}.asc" - if signature_fpath.exists(): - signature_fpath.unlink() - ctx.run( - "gpg", - "--local-user", - key_id, - "--output", - str(signature_fpath), - "--armor", - "--detach-sign", - "--sign", - str(fpath), - ) + tools.utils.gpg_sign(ctx, key_id, fpath) # Export the GPG key in use - _export_gpg_key(ctx, key_id, repo_path, create_repo_path) + tools.utils.export_gpg_key(ctx, key_id, repo_path, create_repo_path) repo_json = _get_repo_json_file_contents( ctx, bucket_name=bucket_name, repo_path=repo_path, repo_json_path=repo_json_path @@ -902,13 +874,13 @@ def _get_repo_json_file_contents( ctx.info(f"Downloading existing '{repo_json_path.relative_to(repo_path)}' file") size = ret["ContentLength"] with repo_json_path.open("wb") as wfh: - with create_progress_bar(file_progress=True) as progress: + with tools.utils.create_progress_bar(file_progress=True) as progress: task = progress.add_task(description="Downloading...", total=size) s3.download_fileobj( Bucket=bucket_name, Key=str(repo_json_path.relative_to(repo_path)), Fileobj=wfh, - Callback=UpdateProgress(progress, task), + Callback=tools.utils.UpdateProgress(progress, task), ) with repo_json_path.open() as rfh: repo_json = json.load(rfh) @@ -950,11 +922,11 @@ def _publish_repo( Publish packaging repositories. """ if nightly_build: - bucket_name = NIGHTLY_BUCKET_NAME + bucket_name = tools.utils.NIGHTLY_BUCKET_NAME elif stage: - bucket_name = STAGING_BUCKET_NAME + bucket_name = tools.utils.STAGING_BUCKET_NAME else: - bucket_name = RELEASE_BUCKET_NAME + bucket_name = tools.utils.RELEASE_BUCKET_NAME ctx.info("Preparing upload ...") s3 = boto3.client("s3") @@ -991,7 +963,7 @@ def _publish_repo( path = pathlib.Path(dirpath, fpath) to_upload_paths.append(path) - with create_progress_bar() as progress: + with tools.utils.create_progress_bar() as progress: task = progress.add_task( "Deleting directories to override.", total=len(to_delete_paths) ) @@ -1015,48 +987,18 @@ def _publish_repo( relpath = upload_path.relative_to(repo_path) size = upload_path.stat().st_size ctx.info(f" {relpath}") - with create_progress_bar(file_progress=True) as progress: + with tools.utils.create_progress_bar(file_progress=True) as progress: task = progress.add_task(description="Uploading...", total=size) s3.upload_file( str(upload_path), bucket_name, str(relpath), - Callback=UpdateProgress(progress, task), + Callback=tools.utils.UpdateProgress(progress, task), ) except KeyboardInterrupt: pass -class UpdateProgress: - def __init__(self, progress, task): - self.progress = progress - self.task = task - - def __call__(self, chunk_size): - self.progress.update(self.task, advance=chunk_size) - - -def create_progress_bar(file_progress: bool = False, **kwargs): - if file_progress: - return Progress( - TextColumn("[progress.description]{task.description}"), - BarColumn(), - DownloadColumn(), - TransferSpeedColumn(), - TextColumn("eta"), - TimeRemainingColumn(), - **kwargs, - ) - return Progress( - TextColumn( - "[progress.description]{task.description}", table_column=Column(ratio=3) - ), - BarColumn(), - expand=True, - **kwargs, - ) - - def _create_repo_path( repo_path: pathlib.Path, salt_version: str, @@ -1084,25 +1026,6 @@ def _create_repo_path( return create_repo_path -def _export_gpg_key( - ctx: Context, key_id: str, repo_path: pathlib.Path, create_repo_path: pathlib.Path -): - keyfile_gpg = create_repo_path.joinpath(GPG_KEY_FILENAME).with_suffix(".gpg") - if keyfile_gpg.exists(): - keyfile_gpg.unlink() - ctx.info( - f"Exporting GnuPG Key '{key_id}' to {keyfile_gpg.relative_to(repo_path)} ..." - ) - ctx.run("gpg", "--output", str(keyfile_gpg), "--export", key_id) - keyfile_pub = create_repo_path.joinpath(GPG_KEY_FILENAME).with_suffix(".pub") - if keyfile_pub.exists(): - keyfile_pub.unlink() - ctx.info( - f"Exporting GnuPG Key '{key_id}' to {keyfile_pub.relative_to(repo_path)} ..." - ) - ctx.run("gpg", "--armor", "--output", str(keyfile_pub), "--export", key_id) - - def _get_latest_version(*versions: str) -> Version: _versions = [] for version in set(versions): diff --git a/tools/pre_commit.py b/tools/pre_commit.py index 1c39ae0b30c..cdc88302199 100644 --- a/tools/pre_commit.py +++ b/tools/pre_commit.py @@ -5,17 +5,17 @@ These commands are used by pre-commit. from __future__ import annotations import logging -import pathlib import shutil from typing import TYPE_CHECKING, cast from jinja2 import Environment, FileSystemLoader from ptscripts import Context, command_group +import tools.utils + log = logging.getLogger(__name__) -REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent -WORKFLOWS = REPO_ROOT / ".github" / "workflows" +WORKFLOWS = tools.utils.REPO_ROOT / ".github" / "workflows" TEMPLATES = WORKFLOWS / "templates" # Define the command group @@ -81,11 +81,11 @@ def generate_workflows(ctx: Context): workflow_path = WORKFLOWS / template template_path = TEMPLATES / f"{template}.jinja" ctx.info( - f"Generating '{workflow_path.relative_to(REPO_ROOT)}' from " - f"template '{template_path.relative_to(REPO_ROOT)}' ..." + f"Generating '{workflow_path.relative_to(tools.utils.REPO_ROOT)}' from " + f"template '{template_path.relative_to(tools.utils.REPO_ROOT)}' ..." ) context = { - "template": template_path.relative_to(REPO_ROOT), + "template": template_path.relative_to(tools.utils.REPO_ROOT), "workflow_name": workflow_name, "includes": includes, "conclusion_needs": NeedsTracker(), diff --git a/tools/utils.py b/tools/utils.py new file mode 100644 index 00000000000..933548efae7 --- /dev/null +++ b/tools/utils.py @@ -0,0 +1,88 @@ +# pylint: disable=resource-leakage,broad-except +from __future__ import annotations + +import pathlib + +from ptscripts import Context +from rich.progress import ( + BarColumn, + Column, + DownloadColumn, + Progress, + TextColumn, + TimeRemainingColumn, + TransferSpeedColumn, +) + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +GPG_KEY_FILENAME = "SALT-PROJECT-GPG-PUBKEY-2023" +NIGHTLY_BUCKET_NAME = "salt-project-prod-salt-artifacts-nightly" +STAGING_BUCKET_NAME = "salt-project-prod-salt-artifacts-staging" +RELEASE_BUCKET_NAME = "salt-project-prod-salt-artifacts-release" + + +class UpdateProgress: + def __init__(self, progress, task): + self.progress = progress + self.task = task + + def __call__(self, chunk_size): + self.progress.update(self.task, advance=chunk_size) + + +def create_progress_bar(file_progress: bool = False, **kwargs): + if file_progress: + return Progress( + TextColumn("[progress.description]{task.description}"), + BarColumn(), + DownloadColumn(), + TransferSpeedColumn(), + TextColumn("eta"), + TimeRemainingColumn(), + **kwargs, + ) + return Progress( + TextColumn( + "[progress.description]{task.description}", table_column=Column(ratio=3) + ), + BarColumn(), + expand=True, + **kwargs, + ) + + +def export_gpg_key( + ctx: Context, key_id: str, repo_path: pathlib.Path, create_repo_path: pathlib.Path +): + keyfile_gpg = create_repo_path.joinpath(GPG_KEY_FILENAME).with_suffix(".gpg") + if keyfile_gpg.exists(): + keyfile_gpg.unlink() + ctx.info( + f"Exporting GnuPG Key '{key_id}' to {keyfile_gpg.relative_to(repo_path)} ..." + ) + ctx.run("gpg", "--output", str(keyfile_gpg), "--export", key_id) + keyfile_pub = create_repo_path.joinpath(GPG_KEY_FILENAME).with_suffix(".pub") + if keyfile_pub.exists(): + keyfile_pub.unlink() + ctx.info( + f"Exporting GnuPG Key '{key_id}' to {keyfile_pub.relative_to(repo_path)} ..." + ) + ctx.run("gpg", "--armor", "--output", str(keyfile_pub), "--export", key_id) + + +def gpg_sign(ctx: Context, key_id: str, path: pathlib.Path): + ctx.info(f"GPG Signing '{path}' ...") + signature_fpath = path.parent / f"{path.name}.asc" + if signature_fpath.exists(): + signature_fpath.unlink() + ctx.run( + "gpg", + "--local-user", + key_id, + "--output", + str(signature_fpath), + "--armor", + "--detach-sign", + "--sign", + str(path), + ) diff --git a/tools/vm.py b/tools/vm.py index 8e1eed6cac5..1df5bbb4078 100644 --- a/tools/vm.py +++ b/tools/vm.py @@ -22,6 +22,8 @@ from typing import TYPE_CHECKING, cast from ptscripts import Context, command_group +import tools.utils + try: import attr import boto3 @@ -52,12 +54,11 @@ if TYPE_CHECKING: log = logging.getLogger(__name__) -REPO_ROOT = pathlib.Path(__file__).parent.parent -STATE_DIR = REPO_ROOT / ".vms-state" -with REPO_ROOT.joinpath("cicd", "golden-images.json").open() as rfh: +STATE_DIR = tools.utils.REPO_ROOT / ".vms-state" +with tools.utils.REPO_ROOT.joinpath("cicd", "golden-images.json").open() as rfh: AMIS = json.load(rfh) REPO_CHECKOUT_ID = hashlib.sha256( - "|".join(list(platform.uname()) + [str(REPO_ROOT)]).encode() + "|".join(list(platform.uname()) + [str(tools.utils.REPO_ROOT)]).encode() ).hexdigest() AWS_REGION = ( os.environ.get("AWS_DEFAULT_REGION") or os.environ.get("AWS_REGION") or "us-west-2" @@ -982,7 +983,7 @@ class VM: "--exclude", ".pytest_cache/", "--exclude", - f"{STATE_DIR.relative_to(REPO_ROOT)}{os.path.sep}", + f"{STATE_DIR.relative_to(tools.utils.REPO_ROOT)}{os.path.sep}", "--exclude", "*.py~", # We need to include artifacts/ to be able to include artifacts/salt @@ -999,7 +1000,7 @@ class VM: # symlink with a copy of what's getting symlinked. rsync_flags.append("--copy-links") # Local repo path - source = f"{REPO_ROOT}{os.path.sep}" + source = f"{tools.utils.REPO_ROOT}{os.path.sep}" # Remote repo path remote_path = self.upload_path.as_posix() if self.is_windows: @@ -1014,7 +1015,7 @@ class VM: return write_env = {k: str(v) for (k, v) in env.items()} write_env_filename = ".ci-env" - write_env_filepath = REPO_ROOT / ".ci-env" + write_env_filepath = tools.utils.REPO_ROOT / ".ci-env" write_env_filepath.write_text(json.dumps(write_env)) # Local path @@ -1241,7 +1242,7 @@ class VM: _ssh_command_args = [ ssh, "-F", - str(self.ssh_config_file.relative_to(REPO_ROOT)), + str(self.ssh_config_file.relative_to(tools.utils.REPO_ROOT)), ] if ssh_options: _ssh_command_args.extend(ssh_options)