tools.utils is now a package

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

tools/pkg/repo/create.py

@@ -19,12 +19,11 @@ from ptscripts import Context, command_group
 
 import tools.pkg
 import tools.utils
-from tools.utils import (
+from tools.utils import Version, parse_versions
-    Version,
+from tools.utils.repo import (
     create_full_repo_path,
     create_top_level_repo_path,
     get_repo_json_file_contents,
-    parse_versions,
 )
 
 try:

tools/pkg/repo/publish.py

@@ -20,13 +20,9 @@ from ptscripts import Context, command_group
 
 import tools.pkg
 import tools.utils
-from tools.utils import (
+import tools.utils.repo
-    Version,
+from tools.utils import Version, get_salt_releases, parse_versions
-    create_full_repo_path,
+from tools.utils.repo import create_full_repo_path, get_repo_json_file_contents
-    get_repo_json_file_contents,
-    get_salt_releases,
-    parse_versions,
-)
 
 try:
     import boto3
@@ -336,7 +332,7 @@ def release(ctx: Context, salt_version: str):
                         Bucket=bucket_name,
                         Key=path,
                         Fileobj=wfh,
-                        Callback=tools.utils.UpdateProgress(progress, task),
+                        Callback=tools.utils.repo.UpdateProgress(progress, task),
                     )
                 updated_contents = re.sub(
                     r"^(baseurl|gpgkey)=https://([^/]+)/(.*)$",
@@ -366,7 +362,7 @@ def release(ctx: Context, salt_version: str):
                         str(upload_path),
                         tools.utils.RELEASE_BUCKET_NAME,
                         str(relpath),
-                        Callback=tools.utils.UpdateProgress(progress, task),
+                        Callback=tools.utils.repo.UpdateProgress(progress, task),
                     )
 
 
@@ -438,7 +434,7 @@ def github(
                 Bucket=tools.utils.STAGING_BUCKET_NAME,
                 Key=str(entry_path),
                 Fileobj=wfh,
-                Callback=tools.utils.UpdateProgress(progress, task),
+                Callback=tools.utils.repo.UpdateProgress(progress, task),
             )
 
     for artifact in artifacts_path.iterdir():
@@ -618,7 +614,7 @@ def _publish_repo(
                     str(upload_path),
                     bucket_name,
                     str(relpath),
-                    Callback=tools.utils.UpdateProgress(progress, task),
+                    Callback=tools.utils.repo.UpdateProgress(progress, task),
                     ExtraArgs={
                         "Metadata": {
                             "x-amz-meta-salt-release-version": salt_version,

tools/utils/__init__.py

@@ -25,19 +25,7 @@ from rich.progress import (
     TransferSpeedColumn,
 )
 
-try:
+REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent
-    import boto3
-    from botocore.exceptions import ClientError
-except ImportError:
-    print(
-        "\nPlease run 'python -m pip install -r "
-        "requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info),
-        file=sys.stderr,
-        flush=True,
-    )
-    raise
-
-REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent
 GPG_KEY_FILENAME = "SALT-PROJECT-GPG-PUBKEY-2023"
 SPB_ENVIRONMENT = os.environ.get("SPB_ENVIRONMENT") or "test"
 STAGING_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-staging"
@@ -45,15 +33,6 @@ RELEASE_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-release"
 BACKUP_BUCKET_NAME = f"salt-project-{SPB_ENVIRONMENT}-salt-artifacts-backup"
 
 
-class UpdateProgress:
-    def __init__(self, progress, task):
-        self.progress = progress
-        self.task = task
-
-    def __call__(self, chunk_size):
-        self.progress.update(self.task, advance=chunk_size)
-
-
 def create_progress_bar(file_progress: bool = False, **kwargs):
     if file_progress:
         return Progress(
@@ -203,106 +182,6 @@ def parse_versions(*versions: str) -> list[Version]:
     return _versions
 
 
-def get_repo_json_file_contents(
-    ctx: Context,
-    bucket_name: str,
-    repo_path: pathlib.Path,
-    repo_json_path: pathlib.Path,
-) -> dict[str, Any]:
-    s3 = boto3.client("s3")
-    repo_json: dict[str, Any] = {}
-    try:
-        ret = s3.head_object(
-            Bucket=bucket_name, Key=str(repo_json_path.relative_to(repo_path))
-        )
-        ctx.info(
-            f"Downloading existing '{repo_json_path.relative_to(repo_path)}' file "
-            f"from bucket {bucket_name}"
-        )
-        size = ret["ContentLength"]
-        with repo_json_path.open("wb") as wfh:
-            with create_progress_bar(file_progress=True) as progress:
-                task = progress.add_task(description="Downloading...", total=size)
-            s3.download_fileobj(
-                Bucket=bucket_name,
-                Key=str(repo_json_path.relative_to(repo_path)),
-                Fileobj=wfh,
-                Callback=UpdateProgress(progress, task),
-            )
-        with repo_json_path.open() as rfh:
-            repo_json = json.load(rfh)
-    except ClientError as exc:
-        if "Error" not in exc.response:
-            raise
-        if exc.response["Error"]["Code"] != "404":
-            raise
-        ctx.info(f"Could not find {repo_json_path} in bucket {bucket_name}")
-    if repo_json:
-        ctx.print(repo_json, soft_wrap=True)
-    return repo_json
-
-
-def create_top_level_repo_path(
-    ctx: Context,
-    repo_path: pathlib.Path,
-    salt_version: str,
-    distro: str,
-    distro_version: str | None = None,  # pylint: disable=bad-whitespace
-    distro_arch: str | None = None,  # pylint: disable=bad-whitespace
-    nightly_build_from: str | None = None,  # pylint: disable=bad-whitespace
-):
-    create_repo_path = repo_path
-    if nightly_build_from:
-        create_repo_path = (
-            create_repo_path
-            / "salt-dev"
-            / nightly_build_from
-            / datetime.utcnow().strftime("%Y-%m-%d")
-        )
-        create_repo_path.mkdir(exist_ok=True, parents=True)
-        with ctx.chdir(create_repo_path.parent):
-            latest_nightly_symlink = pathlib.Path("latest")
-            if not latest_nightly_symlink.exists():
-                ctx.info(
-                    f"Creating 'latest' symlink to '{create_repo_path.relative_to(repo_path)}' ..."
-                )
-                latest_nightly_symlink.symlink_to(
-                    create_repo_path.name, target_is_directory=True
-                )
-    elif "rc" in salt_version:
-        create_repo_path = create_repo_path / "salt_rc"
-    create_repo_path = create_repo_path / "salt" / "py3" / distro
-    if distro_version:
-        create_repo_path = create_repo_path / distro_version
-    if distro_arch:
-        create_repo_path = create_repo_path / distro_arch
-    create_repo_path.mkdir(exist_ok=True, parents=True)
-    return create_repo_path
-
-
-def create_full_repo_path(
-    ctx: Context,
-    repo_path: pathlib.Path,
-    salt_version: str,
-    distro: str,
-    distro_version: str | None = None,  # pylint: disable=bad-whitespace
-    distro_arch: str | None = None,  # pylint: disable=bad-whitespace
-    nightly_build_from: str | None = None,  # pylint: disable=bad-whitespace
-):
-    create_repo_path = create_top_level_repo_path(
-        ctx,
-        repo_path,
-        salt_version,
-        distro,
-        distro_version,
-        distro_arch,
-        nightly_build_from=nightly_build_from,
-    )
-    create_repo_path = create_repo_path / "minor" / salt_version
-    create_repo_path.mkdir(exist_ok=True, parents=True)
-    return create_repo_path
-
-
 def get_file_checksum(fpath: pathlib.Path, hash_name: str) -> str:
     with fpath.open("rb") as rfh:
         try:

tools/utils/repo.py

@@ -0,0 +1,133 @@
+# pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated,bad-whitespace
+from __future__ import annotations
+
+import json
+import pathlib
+import sys
+from datetime import datetime
+from typing import Any
+
+from ptscripts import Context
+
+import tools.utils
+
+try:
+    import boto3
+    from botocore.exceptions import ClientError
+except ImportError:
+    print(
+        "\nPlease run 'python -m pip install -r "
+        "requirements/static/ci/py{}.{}/tools.txt'\n".format(*sys.version_info),
+        file=sys.stderr,
+        flush=True,
+    )
+    raise
+
+
+class UpdateProgress:
+    def __init__(self, progress, task):
+        self.progress = progress
+        self.task = task
+
+    def __call__(self, chunk_size):
+        self.progress.update(self.task, advance=chunk_size)
+
+
+def get_repo_json_file_contents(
+    ctx: Context,
+    bucket_name: str,
+    repo_path: pathlib.Path,
+    repo_json_path: pathlib.Path,
+) -> dict[str, Any]:
+    s3 = boto3.client("s3")
+    repo_json: dict[str, Any] = {}
+    try:
+        ret = s3.head_object(
+            Bucket=bucket_name, Key=str(repo_json_path.relative_to(repo_path))
+        )
+        ctx.info(
+            f"Downloading existing '{repo_json_path.relative_to(repo_path)}' file "
+            f"from bucket {bucket_name}"
+        )
+        size = ret["ContentLength"]
+        with repo_json_path.open("wb") as wfh:
+            with tools.utils.create_progress_bar(file_progress=True) as progress:
+                task = progress.add_task(description="Downloading...", total=size)
+            s3.download_fileobj(
+                Bucket=bucket_name,
+                Key=str(repo_json_path.relative_to(repo_path)),
+                Fileobj=wfh,
+                Callback=UpdateProgress(progress, task),
+            )
+        with repo_json_path.open() as rfh:
+            repo_json = json.load(rfh)
+    except ClientError as exc:
+        if "Error" not in exc.response:
+            raise
+        if exc.response["Error"]["Code"] != "404":
+            raise
+        ctx.info(f"Could not find {repo_json_path} in bucket {bucket_name}")
+    if repo_json:
+        ctx.print(repo_json, soft_wrap=True)
+    return repo_json
+
+
+def create_top_level_repo_path(
+    ctx: Context,
+    repo_path: pathlib.Path,
+    salt_version: str,
+    distro: str,
+    distro_version: str | None = None,  # pylint: disable=bad-whitespace
+    distro_arch: str | None = None,  # pylint: disable=bad-whitespace
+    nightly_build_from: str | None = None,  # pylint: disable=bad-whitespace
+):
+    create_repo_path = repo_path
+    if nightly_build_from:
+        create_repo_path = (
+            create_repo_path
+            / "salt-dev"
+            / nightly_build_from
+            / datetime.utcnow().strftime("%Y-%m-%d")
+        )
+        create_repo_path.mkdir(exist_ok=True, parents=True)
+        with ctx.chdir(create_repo_path.parent):
+            latest_nightly_symlink = pathlib.Path("latest")
+            if not latest_nightly_symlink.exists():
+                ctx.info(
+                    f"Creating 'latest' symlink to '{create_repo_path.relative_to(repo_path)}' ..."
+                )
+                latest_nightly_symlink.symlink_to(
+                    create_repo_path.name, target_is_directory=True
+                )
+    elif "rc" in salt_version:
+        create_repo_path = create_repo_path / "salt_rc"
+    create_repo_path = create_repo_path / "salt" / "py3" / distro
+    if distro_version:
+        create_repo_path = create_repo_path / distro_version
+    if distro_arch:
+        create_repo_path = create_repo_path / distro_arch
+    create_repo_path.mkdir(exist_ok=True, parents=True)
+    return create_repo_path
+
+
+def create_full_repo_path(
+    ctx: Context,
+    repo_path: pathlib.Path,
+    salt_version: str,
+    distro: str,
+    distro_version: str | None = None,  # pylint: disable=bad-whitespace
+    distro_arch: str | None = None,  # pylint: disable=bad-whitespace
+    nightly_build_from: str | None = None,  # pylint: disable=bad-whitespace
+):
+    create_repo_path = create_top_level_repo_path(
+        ctx,
+        repo_path,
+        salt_version,
+        distro,
+        distro_version,
+        distro_arch,
+        nightly_build_from=nightly_build_from,
+    )
+    create_repo_path = create_repo_path / "minor" / salt_version
+    create_repo_path.mkdir(exist_ok=True, parents=True)
+    return create_repo_path