From 4f30001ca5e37c44af99a998225d12b97f29426c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 16 Aug 2023 14:35:23 +0100 Subject: [PATCH] `tools.utils` is now a package Signed-off-by: Pedro Algarvio --- tools/pkg/repo/create.py | 5 +- tools/pkg/repo/publish.py | 18 ++-- tools/{utils.py => utils/__init__.py} | 123 +----------------------- tools/utils/repo.py | 133 ++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 136 deletions(-) rename tools/{utils.py => utils/__init__.py} (71%) create mode 100644 tools/utils/repo.py diff --git a/tools/pkg/repo/create.py b/tools/pkg/repo/create.py index 904208d53da..f8ca561295c 100644 --- a/tools/pkg/repo/create.py +++ b/tools/pkg/repo/create.py @@ -19,12 +19,11 @@ from ptscripts import Context, command_group import tools.pkg import tools.utils -from tools.utils import ( - Version, +from tools.utils import Version, parse_versions +from tools.utils.repo import ( create_full_repo_path, create_top_level_repo_path, get_repo_json_file_contents, - parse_versions, ) try: diff --git a/tools/pkg/repo/publish.py b/tools/pkg/repo/publish.py index 1c87d20b490..56155b94a59 100644 --- a/tools/pkg/repo/publish.py +++ b/tools/pkg/repo/publish.py @@ -20,13 +20,9 @@ from ptscripts import Context, command_group import tools.pkg import tools.utils -from tools.utils import ( - Version, - create_full_repo_path, - get_repo_json_file_contents, - get_salt_releases, - parse_versions, -) +import tools.utils.repo +from tools.utils import Version, get_salt_releases, parse_versions +from tools.utils.repo import create_full_repo_path, get_repo_json_file_contents try: import boto3 @@ -336,7 +332,7 @@ def release(ctx: Context, salt_version: str): Bucket=bucket_name, Key=path, Fileobj=wfh, - Callback=tools.utils.UpdateProgress(progress, task), + Callback=tools.utils.repo.UpdateProgress(progress, task), ) updated_contents = re.sub( r"^(baseurl|gpgkey)=https://([^/]+)/(.*)$", @@ -366,7 +362,7 @@ def release(ctx: Context, salt_version: str): str(upload_path), tools.utils.RELEASE_BUCKET_NAME, str(relpath), - Callback=tools.utils.UpdateProgress(progress, task), + Callback=tools.utils.repo.UpdateProgress(progress, task), ) @@ -438,7 +434,7 @@ def github( Bucket=tools.utils.STAGING_BUCKET_NAME, Key=str(entry_path), Fileobj=wfh, - Callback=tools.utils.UpdateProgress(progress, task), + Callback=tools.utils.repo.UpdateProgress(progress, task), ) for artifact in artifacts_path.iterdir(): @@ -618,7 +614,7 @@ def _publish_repo( str(upload_path), bucket_name, str(relpath), - Callback=tools.utils.UpdateProgress(progress, task), + Callback=tools.utils.repo.UpdateProgress(progress, task), ExtraArgs={ "Metadata": { "x-amz-meta-salt-release-version": salt_version, diff --git a/tools/utils.py b/tools/utils/__init__.py similarity index 71% rename from tools/utils.py rename to tools/utils/__init__.py index 8367607687e..ca3bad19dbc 100644 --- a/tools/utils.py +++ b/tools/utils/__init__.py @@ -25,19 +25,7 @@ from rich.progress import ( TransferSpeedColumn, ) -try: - import boto3 - from botocore.exceptions import ClientError -except ImportError: - print( - "\nPlease run 'python -m pip install -r " - "requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info), - file=sys.stderr, - flush=True, - ) - raise - -REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent GPG_KEY_FILENAME = "SALT-PROJECT-GPG-PUBKEY-2023" SPB_ENVIRONMENT = os.environ.get("SPB_ENVIRONMENT") or "test" STAGING_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-staging" @@ -45,15 +33,6 @@ RELEASE_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-release" BACKUP_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-backup" -class UpdateProgress: - def __init__(self, progress, task): - self.progress = progress - self.task = task - - def __call__(self, chunk_size): - self.progress.update(self.task, advance=chunk_size) - - def create_progress_bar(file_progress: bool = False, **kwargs): if file_progress: return Progress( @@ -203,106 +182,6 @@ def parse_versions(*versions: str) -> list[Version]: return _versions -def get_repo_json_file_contents( - ctx: Context, - bucket_name: str, - repo_path: pathlib.Path, - repo_json_path: pathlib.Path, -) -> dict[str, Any]: - s3 = boto3.client("s3") - repo_json: dict[str, Any] = {} - try: - ret = s3.head_object( - Bucket=bucket_name, Key=str(repo_json_path.relative_to(repo_path)) - ) - ctx.info( - f"Downloading existing '{repo_json_path.relative_to(repo_path)}' file " - f"from bucket {bucket_name}" - ) - size = ret["ContentLength"] - with repo_json_path.open("wb") as wfh: - with create_progress_bar(file_progress=True) as progress: - task = progress.add_task(description="Downloading...", total=size) - s3.download_fileobj( - Bucket=bucket_name, - Key=str(repo_json_path.relative_to(repo_path)), - Fileobj=wfh, - Callback=UpdateProgress(progress, task), - ) - with repo_json_path.open() as rfh: - repo_json = json.load(rfh) - except ClientError as exc: - if "Error" not in exc.response: - raise - if exc.response["Error"]["Code"] != "404": - raise - ctx.info(f"Could not find {repo_json_path} in bucket {bucket_name}") - if repo_json: - ctx.print(repo_json, soft_wrap=True) - return repo_json - - -def create_top_level_repo_path( - ctx: Context, - repo_path: pathlib.Path, - salt_version: str, - distro: str, - distro_version: str | None = None, # pylint: disable=bad-whitespace - distro_arch: str | None = None, # pylint: disable=bad-whitespace - nightly_build_from: str | None = None, # pylint: disable=bad-whitespace -): - create_repo_path = repo_path - if nightly_build_from: - create_repo_path = ( - create_repo_path - / "salt-dev" - / nightly_build_from - / datetime.utcnow().strftime("%Y-%m-%d") - ) - create_repo_path.mkdir(exist_ok=True, parents=True) - with ctx.chdir(create_repo_path.parent): - latest_nightly_symlink = pathlib.Path("latest") - if not latest_nightly_symlink.exists(): - ctx.info( - f"Creating 'latest' symlink to '{create_repo_path.relative_to(repo_path)}' ..." - ) - latest_nightly_symlink.symlink_to( - create_repo_path.name, target_is_directory=True - ) - elif "rc" in salt_version: - create_repo_path = create_repo_path / "salt_rc" - create_repo_path = create_repo_path / "salt" / "py3" / distro - if distro_version: - create_repo_path = create_repo_path / distro_version - if distro_arch: - create_repo_path = create_repo_path / distro_arch - create_repo_path.mkdir(exist_ok=True, parents=True) - return create_repo_path - - -def create_full_repo_path( - ctx: Context, - repo_path: pathlib.Path, - salt_version: str, - distro: str, - distro_version: str | None = None, # pylint: disable=bad-whitespace - distro_arch: str | None = None, # pylint: disable=bad-whitespace - nightly_build_from: str | None = None, # pylint: disable=bad-whitespace -): - create_repo_path = create_top_level_repo_path( - ctx, - repo_path, - salt_version, - distro, - distro_version, - distro_arch, - nightly_build_from=nightly_build_from, - ) - create_repo_path = create_repo_path / "minor" / salt_version - create_repo_path.mkdir(exist_ok=True, parents=True) - return create_repo_path - - def get_file_checksum(fpath: pathlib.Path, hash_name: str) -> str: with fpath.open("rb") as rfh: try: diff --git a/tools/utils/repo.py b/tools/utils/repo.py new file mode 100644 index 00000000000..075f86ab0c9 --- /dev/null +++ b/tools/utils/repo.py @@ -0,0 +1,133 @@ +# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated,bad-whitespace +from __future__ import annotations + +import json +import pathlib +import sys +from datetime import datetime +from typing import Any + +from ptscripts import Context + +import tools.utils + +try: + import boto3 + from botocore.exceptions import ClientError +except ImportError: + print( + "\nPlease run 'python -m pip install -r " + "requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info), + file=sys.stderr, + flush=True, + ) + raise + + +class UpdateProgress: + def __init__(self, progress, task): + self.progress = progress + self.task = task + + def __call__(self, chunk_size): + self.progress.update(self.task, advance=chunk_size) + + +def get_repo_json_file_contents( + ctx: Context, + bucket_name: str, + repo_path: pathlib.Path, + repo_json_path: pathlib.Path, +) -> dict[str, Any]: + s3 = boto3.client("s3") + repo_json: dict[str, Any] = {} + try: + ret = s3.head_object( + Bucket=bucket_name, Key=str(repo_json_path.relative_to(repo_path)) + ) + ctx.info( + f"Downloading existing '{repo_json_path.relative_to(repo_path)}' file " + f"from bucket {bucket_name}" + ) + size = ret["ContentLength"] + with repo_json_path.open("wb") as wfh: + with tools.utils.create_progress_bar(file_progress=True) as progress: + task = progress.add_task(description="Downloading...", total=size) + s3.download_fileobj( + Bucket=bucket_name, + Key=str(repo_json_path.relative_to(repo_path)), + Fileobj=wfh, + Callback=UpdateProgress(progress, task), + ) + with repo_json_path.open() as rfh: + repo_json = json.load(rfh) + except ClientError as exc: + if "Error" not in exc.response: + raise + if exc.response["Error"]["Code"] != "404": + raise + ctx.info(f"Could not find {repo_json_path} in bucket {bucket_name}") + if repo_json: + ctx.print(repo_json, soft_wrap=True) + return repo_json + + +def create_top_level_repo_path( + ctx: Context, + repo_path: pathlib.Path, + salt_version: str, + distro: str, + distro_version: str | None = None, # pylint: disable=bad-whitespace + distro_arch: str | None = None, # pylint: disable=bad-whitespace + nightly_build_from: str | None = None, # pylint: disable=bad-whitespace +): + create_repo_path = repo_path + if nightly_build_from: + create_repo_path = ( + create_repo_path + / "salt-dev" + / nightly_build_from + / datetime.utcnow().strftime("%Y-%m-%d") + ) + create_repo_path.mkdir(exist_ok=True, parents=True) + with ctx.chdir(create_repo_path.parent): + latest_nightly_symlink = pathlib.Path("latest") + if not latest_nightly_symlink.exists(): + ctx.info( + f"Creating 'latest' symlink to '{create_repo_path.relative_to(repo_path)}' ..." + ) + latest_nightly_symlink.symlink_to( + create_repo_path.name, target_is_directory=True + ) + elif "rc" in salt_version: + create_repo_path = create_repo_path / "salt_rc" + create_repo_path = create_repo_path / "salt" / "py3" / distro + if distro_version: + create_repo_path = create_repo_path / distro_version + if distro_arch: + create_repo_path = create_repo_path / distro_arch + create_repo_path.mkdir(exist_ok=True, parents=True) + return create_repo_path + + +def create_full_repo_path( + ctx: Context, + repo_path: pathlib.Path, + salt_version: str, + distro: str, + distro_version: str | None = None, # pylint: disable=bad-whitespace + distro_arch: str | None = None, # pylint: disable=bad-whitespace + nightly_build_from: str | None = None, # pylint: disable=bad-whitespace +): + create_repo_path = create_top_level_repo_path( + ctx, + repo_path, + salt_version, + distro, + distro_version, + distro_arch, + nightly_build_from=nightly_build_from, + ) + create_repo_path = create_repo_path / "minor" / salt_version + create_repo_path.mkdir(exist_ok=True, parents=True) + return create_repo_path