Create get_cicd_shared_context() and get_golden_images()

tools/pkg/build.py

@@ -14,7 +14,6 @@ import tarfile
 import zipfile
 from typing import TYPE_CHECKING
 
-import yaml
 from ptscripts import Context, command_group
 
 import tools.utils
@@ -30,13 +29,6 @@ build = command_group(
 )
 
 
-def _get_shared_constants():
-    shared_constants = (
-        tools.utils.REPO_ROOT / "cicd" / "shared-gh-workflows-context.yml"
-    )
-    return yaml.safe_load(shared_constants.read_text())
-
-
 @build.command(
     name="deb",
     arguments={
@@ -80,7 +72,7 @@ def debian(
             )
             ctx.exit(1)
         ctx.info("Building the package from the source files")
-        shared_constants = _get_shared_constants()
+        shared_constants = tools.utils.get_cicd_shared_context()
         if not python_version:
             python_version = shared_constants["python_version"]
         if not relenv_version:
@@ -152,7 +144,7 @@ def rpm(
             )
             ctx.exit(1)
         ctx.info("Building the package from the source files")
-        shared_constants = _get_shared_constants()
+        shared_constants = tools.utils.get_cicd_shared_context()
         if not python_version:
             python_version = shared_constants["python_version"]
         if not relenv_version:
@@ -237,7 +229,7 @@ def macos(
 
     if not onedir:
         # Prep the salt onedir if not building from an existing one
-        shared_constants = _get_shared_constants()
+        shared_constants = tools.utils.get_cicd_shared_context()
         if not python_version:
             python_version = shared_constants["python_version"]
         if not relenv_version:
@@ -326,7 +318,7 @@ def windows(
         assert salt_version is not None
         assert arch is not None
 
-    shared_constants = _get_shared_constants()
+    shared_constants = tools.utils.get_cicd_shared_context()
     if not python_version:
         python_version = shared_constants["python_version"]
     if not relenv_version:
@@ -493,7 +485,7 @@ def onedir_dependencies(
     if platform != "macos" and arch == "arm64":
         arch = "aarch64"
 
-    shared_constants = _get_shared_constants()
+    shared_constants = tools.utils.get_cicd_shared_context()
     if not python_version:
         python_version = shared_constants["python_version"]
     if not relenv_version:
@@ -632,7 +624,7 @@ def salt_onedir(
     if platform == "darwin":
         platform = "macos"
 
-    shared_constants = _get_shared_constants()
+    shared_constants = tools.utils.get_cicd_shared_context()
     if not relenv_version:
         relenv_version = shared_constants["relenv_version"]
     if TYPE_CHECKING:

tools/precommit/workflows.py

@@ -5,12 +5,10 @@ These commands are used for our GitHub Actions workflows.
 # pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated
 from __future__ import annotations
 
-import json
 import logging
 import shutil
 from typing import TYPE_CHECKING, cast
 
-import yaml
 from jinja2 import Environment, FileSystemLoader, StrictUndefined
 from ptscripts import Context, command_group
 
@@ -20,10 +18,6 @@ log = logging.getLogger(__name__)
 
 WORKFLOWS = tools.utils.REPO_ROOT / ".github" / "workflows"
 TEMPLATES = WORKFLOWS / "templates"
-with tools.utils.REPO_ROOT.joinpath("cicd", "golden-images.json").open(
-    "r", encoding="utf-8"
-) as rfh:
-    AMIS = json.load(rfh)
 
 
 # Define the command group
@@ -216,7 +210,7 @@ def generate_workflows(ctx: Context):
         "opensuse-15",
         "windows",
     )
-    for slug in sorted(AMIS):
+    for slug in sorted(tools.utils.get_golden_images()):
         if slug.startswith(linux_skip_pkg_download_tests):
             continue
         if "arm64" in slug:
@@ -251,7 +245,7 @@ def generate_workflows(ctx: Context):
         "photon": [],
         "redhat": [],
     }
-    for slug in sorted(AMIS):
+    for slug in sorted(tools.utils.get_golden_images()):
         if slug.endswith("-arm64"):
             continue
         if not slug.startswith(
@@ -274,7 +268,7 @@ def generate_workflows(ctx: Context):
                 build_rpms_listing.append((distro, release, arch))
 
     build_debs_listing = []
-    for slug in sorted(AMIS):
+    for slug in sorted(tools.utils.get_golden_images()):
         if not slug.startswith(("debian-", "ubuntu-")):
             continue
         if slug.endswith("-arm64"):
@@ -335,10 +329,7 @@ def generate_workflows(ctx: Context):
                 "windows-2022",
             ),
         }
-        shared_context_file = (
-            tools.utils.REPO_ROOT / "cicd" / "shared-gh-workflows-context.yml"
-        )
-        shared_context = yaml.safe_load(shared_context_file.read_text())
+        shared_context = tools.utils.get_cicd_shared_context()
         for key, value in shared_context.items():
             context[key] = value
         loaded_template = env.get_template(template_path.name)

tools/testsuite/__init__.py

@@ -5,7 +5,6 @@ These commands are related to the test suite.
 # pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated
 from __future__ import annotations
 
-import json
 import logging
 from typing import TYPE_CHECKING
 
@@ -15,11 +14,6 @@ import tools.utils
 import tools.utils.gh
 from tools.utils import ExitCode
 
-with tools.utils.REPO_ROOT.joinpath("cicd", "golden-images.json").open(
-    "r", encoding="utf-8"
-) as rfh:
-    OS_SLUGS = sorted(json.load(rfh))
-
 log = logging.getLogger(__name__)
 
 # Define the command group
@@ -57,7 +51,7 @@ ts = command_group(name="ts", help="Test Suite Related Commands", description=__
         "slug": {
             "help": "The OS slug",
             "required": True,
-            "choices": OS_SLUGS,
+            "choices": sorted(tools.utils.get_golden_images()),
         },
         "pkg": {
             "help": "Also download package test artifacts",

tools/testsuite/download.py

@@ -5,7 +5,6 @@ These commands are related to downloading test suite CI artifacts.
 # pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated
 from __future__ import annotations
 
-import json
 import logging
 import pathlib
 from typing import TYPE_CHECKING
@@ -15,11 +14,6 @@ from ptscripts import Context, command_group
 import tools.utils
 import tools.utils.gh
 
-with tools.utils.REPO_ROOT.joinpath("cicd", "golden-images.json").open(
-    "r", encoding="utf-8"
-) as rfh:
-    OS_SLUGS = sorted(json.load(rfh))
-
 log = logging.getLogger(__name__)
 
 
@@ -135,7 +129,7 @@ def download_nox_artifact(
         "slug": {
             "help": "The OS slug",
             "required": True,
-            "choices": OS_SLUGS,
+            "choices": sorted(tools.utils.get_golden_images()),
         },
         "repository": {
             "help": "The repository to query, e.g. saltstack/salt",

tools/utils/__init__.py

@@ -1,22 +1,17 @@
 # pylint: disable=resource-leakage,broad-except,3rd-party-module-not-gated,bad-whitespace
 from __future__ import annotations
 
-import fnmatch
 import hashlib
 import json
 import os
 import pathlib
 import shutil
 import sys
-import tempfile
-import zipfile
-from datetime import datetime
 from enum import IntEnum
-from typing import Any
+from functools import cache
 
-import boto3
 import packaging.version
-from botocore.exceptions import ClientError
+import yaml
 from ptscripts import Context
 from rich.progress import (
     BarColumn,
@@ -284,3 +279,23 @@ def get_platform_and_arch_from_slug(slug: str) -> tuple[str, str]:
         else:
             arch = "x86_64"
     return platform, arch
+
+
+@cache
+def get_cicd_shared_context():
+    """
+    Return the CI/CD shared context file contents.
+    """
+    shared_context_file = REPO_ROOT / "cicd" / "shared-gh-workflows-context.yml"
+    return yaml.safe_load(shared_context_file.read_text())
+
+
+@cache
+def get_golden_images():
+    """
+    Return the golden images information stored on file.
+    """
+    with REPO_ROOT.joinpath("cicd", "golden-images.json").open(
+        "r", encoding="utf-8"
+    ) as rfh:
+        return json.load(rfh)

tools/vm.py

@@ -47,10 +47,6 @@ if TYPE_CHECKING:
 log = logging.getLogger(__name__)
 
 STATE_DIR = tools.utils.REPO_ROOT / ".vms-state"
-with tools.utils.REPO_ROOT.joinpath("cicd", "golden-images.json").open(
-    "r", encoding="utf-8"
-) as rfh:
-    AMIS = json.load(rfh)
 REPO_CHECKOUT_ID = hashlib.sha256(
     "|".join(list(platform.uname()) + [str(tools.utils.REPO_ROOT)]).encode()
 ).hexdigest()
@@ -67,7 +63,7 @@ vm.add_argument("--region", help="The AWS region.", default=AWS_REGION)
         "name": {
             "help": "The VM Name",
             "metavar": "VM_NAME",
-            "choices": list(AMIS),
+            "choices": sorted(tools.utils.get_golden_images()),
         },
         "key_name": {
             "help": "The SSH key name. Will default to TOOLS_KEY_NAME in environment",
@@ -791,10 +787,11 @@ class VM:
 
     @config.default
     def _config_default(self):
+        golden_images = tools.utils.get_golden_images()
         config = AMIConfig(
             **{
                 key: value
-                for (key, value) in AMIS[self.name].items()
+                for (key, value) in golden_images[self.name].items()
                 if key in AMIConfig.__annotations__
             }
         )