Only run the download tests for the latest subpath if the current version being released is greater than or equal to the latest salt release

.github/workflows/ci.yml

@@ -37,6 +37,8 @@ jobs:
       testrun: ${{ steps.define-testrun.outputs.testrun }}
       salt-version: ${{ steps.setup-salt-version.outputs.salt-version }}
       cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }}
+      latest-release: ${{ steps.get-salt-releases.outputs.latest-release }}
+      releases: ${{ steps.get-salt-releases.outputs.releases }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -204,6 +206,18 @@ jobs:
         run: |
           echo '${{ steps.define-jobs.outputs.jobs }}' | jq -C '.'
 
+      - name: Get Salt Releases
+        id: get-salt-releases
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          tools ci get-releases
+
+      - name: Check Salt Releases
+        run: |
+          echo '${{ steps.get-salt-releases.outputs.latest-release }}' | jq -C '.'
+          echo '${{ steps.get-salt-releases.outputs.releases }}' | jq -C '.'
+
       - name: Define Testrun
         id: define-testrun
         run: |

.github/workflows/nightly.yml

@@ -90,6 +90,8 @@ jobs:
       testrun: ${{ steps.define-testrun.outputs.testrun }}
       salt-version: ${{ steps.setup-salt-version.outputs.salt-version }}
       cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }}
+      latest-release: ${{ steps.get-salt-releases.outputs.latest-release }}
+      releases: ${{ steps.get-salt-releases.outputs.releases }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -257,6 +259,18 @@ jobs:
         run: |
           echo '${{ steps.define-jobs.outputs.jobs }}' | jq -C '.'
 
+      - name: Get Salt Releases
+        id: get-salt-releases
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          tools ci get-releases
+
+      - name: Check Salt Releases
+        run: |
+          echo '${{ steps.get-salt-releases.outputs.latest-release }}' | jq -C '.'
+          echo '${{ steps.get-salt-releases.outputs.releases }}' | jq -C '.'
+
       - name: Define Testrun
         id: define-testrun
         run: |

.github/workflows/release.yml

@@ -67,6 +67,8 @@ jobs:
     outputs:
       salt-version: ${{ steps.setup-salt-version.outputs.salt-version }}
       cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }}
+      latest-release: ${{ steps.get-salt-releases.outputs.latest-release }}
+      releases: ${{ steps.get-salt-releases.outputs.releases }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -103,6 +105,18 @@ jobs:
         run: |
           tools pkg repo confirm-staged --repository ${{ github.repository }} ${{ steps.setup-salt-version.outputs.salt-version }}
 
+      - name: Get Salt Releases
+        id: get-salt-releases
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          tools ci get-releases
+
+      - name: Check Salt Releases
+        run: |
+          echo '${{ steps.get-salt-releases.outputs.latest-release }}' | jq -C '.'
+          echo '${{ steps.get-salt-releases.outputs.releases }}' | jq -C '.'
+
       - name: Set Cache Seed Output
         id: set-cache-seed
         run: |

.github/workflows/scheduled.yml

@@ -80,6 +80,8 @@ jobs:
       testrun: ${{ steps.define-testrun.outputs.testrun }}
       salt-version: ${{ steps.setup-salt-version.outputs.salt-version }}
       cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }}
+      latest-release: ${{ steps.get-salt-releases.outputs.latest-release }}
+      releases: ${{ steps.get-salt-releases.outputs.releases }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -247,6 +249,18 @@ jobs:
         run: |
           echo '${{ steps.define-jobs.outputs.jobs }}' | jq -C '.'
 
+      - name: Get Salt Releases
+        id: get-salt-releases
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          tools ci get-releases
+
+      - name: Check Salt Releases
+        run: |
+          echo '${{ steps.get-salt-releases.outputs.latest-release }}' | jq -C '.'
+          echo '${{ steps.get-salt-releases.outputs.releases }}' | jq -C '.'
+
       - name: Define Testrun
         id: define-testrun
         run: |

.github/workflows/staging.yml

@@ -81,6 +81,8 @@ jobs:
       testrun: ${{ steps.define-testrun.outputs.testrun }}
       salt-version: ${{ steps.setup-salt-version.outputs.salt-version }}
       cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }}
+      latest-release: ${{ steps.get-salt-releases.outputs.latest-release }}
+      releases: ${{ steps.get-salt-releases.outputs.releases }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -254,6 +256,18 @@ jobs:
         run: |
           echo '${{ steps.define-jobs.outputs.jobs }}' | jq -C '.'
 
+      - name: Get Salt Releases
+        id: get-salt-releases
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          tools ci get-releases
+
+      - name: Check Salt Releases
+        run: |
+          echo '${{ steps.get-salt-releases.outputs.latest-release }}' | jq -C '.'
+          echo '${{ steps.get-salt-releases.outputs.releases }}' | jq -C '.'
+
       - name: Define Testrun
         id: define-testrun
         run: |
@@ -2115,6 +2129,7 @@ jobs:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
       environment: staging
       skip-code-coverage: true
+      latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
     secrets: inherit
 
   almalinux-8-arm64-pkg-download-tests:
@@ -2574,6 +2589,7 @@ jobs:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
       environment: staging
       skip-code-coverage: true
+      latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
     secrets: inherit
 
   windows-2022-nsis-amd64-pkg-download-tests:
@@ -2610,6 +2626,7 @@ jobs:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
       environment: staging
       skip-code-coverage: true
+      latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
     secrets: inherit
 
   publish-pypi:

.github/workflows/templates/layout.yml.jinja

@@ -90,6 +90,8 @@ jobs:
       testrun: ${{ steps.define-testrun.outputs.testrun }}
       salt-version: ${{ steps.setup-salt-version.outputs.salt-version }}
       cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }}
+      latest-release: ${{ steps.get-salt-releases.outputs.latest-release }}
+      releases: ${{ steps.get-salt-releases.outputs.releases }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -267,6 +269,18 @@ jobs:
         run: |
           echo '${{ steps.define-jobs.outputs.jobs }}' | jq -C '.'
 
+      - name: Get Salt Releases
+        id: get-salt-releases
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          tools ci get-releases
+
+      - name: Check Salt Releases
+        run: |
+          echo '${{ steps.get-salt-releases.outputs.latest-release }}' | jq -C '.'
+          echo '${{ steps.get-salt-releases.outputs.releases }}' | jq -C '.'
+
       - name: Define Testrun
         id: define-testrun
         run: |

.github/workflows/templates/release.yml.jinja

@@ -95,6 +95,8 @@ permissions:
     outputs:
       salt-version: ${{ steps.setup-salt-version.outputs.salt-version }}
       cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }}
+      latest-release: ${{ steps.get-salt-releases.outputs.latest-release }}
+      releases: ${{ steps.get-salt-releases.outputs.releases }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -131,6 +133,18 @@ permissions:
         run: |
           tools pkg repo confirm-staged --repository ${{ github.repository }} ${{ steps.setup-salt-version.outputs.salt-version }}
 
+      - name: Get Salt Releases
+        id: get-salt-releases
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          tools ci get-releases
+
+      - name: Check Salt Releases
+        run: |
+          echo '${{ steps.get-salt-releases.outputs.latest-release }}' | jq -C '.'
+          echo '${{ steps.get-salt-releases.outputs.releases }}' | jq -C '.'
+
       - name: Set Cache Seed Output
         id: set-cache-seed
         run: |

.github/workflows/templates/test-pkg-repo-downloads.yml.jinja

@@ -56,6 +56,7 @@
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
       environment: <{ gh_environment }>
       skip-code-coverage: true
+      latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
     secrets: inherit
 
   <%- endfor %>
@@ -89,6 +90,7 @@
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
       environment: <{ gh_environment }>
       skip-code-coverage: true
+      latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
     secrets: inherit
   <%- endfor %>
 
@@ -123,6 +125,7 @@
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
       environment: <{ gh_environment }>
       skip-code-coverage: true
+      latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
     secrets: inherit
     <%- endfor %>
   <%- endfor %>

.github/workflows/test-package-downloads-action-linux.yml

@@ -27,6 +27,10 @@ on:
         required: true
         type: string
         description: The environment to run tests against
+      latest-release:
+        required: true
+        type: string
+        description: The latest salt release
       package-name:
         required: false
         type: string
@@ -216,6 +220,7 @@ jobs:
           SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }}
           SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }}
           SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
+          LATEST_SALT_RELEASE: "${{ inputs.latest-release }}"
         run: |
           tools --timestamps --timeout-secs=1800 vm testplan --skip-requirements-install \
           -E INSTALL_TYPE -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \
@@ -232,6 +237,7 @@ jobs:
           SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }}
           SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }}
           SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
+          LATEST_SALT_RELEASE: "${{ inputs.latest-release }}"
         run: |
           tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
           -E INSTALL_TYPE -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \

.github/workflows/test-package-downloads-action-macos.yml

@@ -27,6 +27,10 @@ on:
         required: true
         type: string
         description: The environment to run tests against
+      latest-release:
+        required: true
+        type: string
+        description: The latest salt release
       python-version:
         required: false
         type: string
@@ -194,6 +198,7 @@ jobs:
           GITHUB_ACTIONS_PIPELINE: "1"
           SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
           SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
+          LATEST_SALT_RELEASE: "${{ inputs.latest-release }}"
         run: |
           sudo -E nox --force-color -e ${{ inputs.nox-session }} -- download-pkgs
 
@@ -209,6 +214,7 @@ jobs:
           SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
           SALT_RELEASE: "${{ inputs.salt-version }}"
           SALT_REPO_ARCH: ${{ inputs.arch }}
+          LATEST_SALT_RELEASE: "${{ inputs.latest-release }}"
           SALT_REPO_TYPE: ${{ inputs.environment }}
           SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }}
           SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }}

.github/workflows/test-package-downloads-action-windows.yml

@@ -31,6 +31,10 @@ on:
         required: true
         type: string
         description: The environment to run tests against
+      latest-release:
+        required: true
+        type: string
+        description: The latest salt release
       package-name:
         required: false
         type: string
@@ -220,6 +224,7 @@ jobs:
           INSTALL_TYPE: ${{ inputs.pkg-type }}
           SALT_RELEASE: "${{ inputs.salt-version }}"
           SALT_REPO_ARCH: ${{ inputs.arch }}
+          LATEST_SALT_RELEASE: "${{ inputs.latest-release }}"
           SALT_REPO_TYPE: ${{ inputs.environment }}
           SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }}
           SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }}
@@ -237,6 +242,7 @@ jobs:
           INSTALL_TYPE: ${{ inputs.pkg-type }}
           SALT_RELEASE: "${{ inputs.salt-version }}"
           SALT_REPO_ARCH: ${{ inputs.arch }}
+          LATEST_SALT_RELEASE: "${{ inputs.latest-release }}"
           SALT_REPO_TYPE: ${{ inputs.environment }}
           SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }}
           SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }}

pkg/tests/download/test_pkg_download.py

@@ -95,9 +95,23 @@ def get_salt_release():
     return salt_release
 
 
+def get_repo_subpath_params():
+    current_release = packaging.version.parse(get_salt_release())
+    params = ["minor", current_release.major]
+    latest_env_var = os.environ.get("LATEST_SALT_RELEASE")
+    if latest_env_var is not None:
+        latest_release = packaging.version.parse(get_salt_release())
+        if current_release >= latest_release:
+            log.debug(
+                f"Running the tests for the latest release since {str(current_release)} >= {str(latest_release)}"
+            )
+            params.append("latest")
+    return params
+
+
 @pytest.fixture(
     scope="module",
-    params=["latest", "minor", packaging.version.parse(get_salt_release()).major],
+    params=[get_repo_subpath_params()],
 )
 def repo_subpath(request):
     return request.param

tools/ci.py

@@ -645,3 +645,29 @@ def pkg_matrix(ctx: Context, distro_slug: str, pkg_type: str):
         with open(github_output, "a", encoding="utf-8") as wfh:
             wfh.write(f"matrix={json.dumps(matrix)}\n")
     ctx.exit(0)
+
+
+@ci.command(
+    name="get-releases",
+    arguments={
+        "repository": {
+            "help": "The repository to query for releases, e.g. saltstack/salt",
+        },
+    },
+)
+def get_releases(ctx: Context, repository: str = "saltstack/salt"):
+    """
+    Generate the latest salt release.
+    """
+    github_output = os.environ.get("GITHUB_OUTPUT")
+
+    if github_output is None:
+        ctx.exit(1, "The 'GITHUB_OUTPUT' variable is not set.")
+    else:
+        releases = tools.utils.get_salt_releases(ctx, repository)
+        latest = releases[-1]
+
+        with open(github_output, "a", encoding="utf-8") as wfh:
+            wfh.write(f"latest-release={latest}\n")
+            wfh.write(f"releases={json.dumps(releases)}\n")
+        ctx.exit(0)

tools/pkg/repo.py

@@ -23,7 +23,7 @@ from ptscripts import Context, command_group
 
 import tools.pkg
 import tools.utils
-from tools.utils import Version
+from tools.utils import Version, get_salt_releases
 
 try:
     import boto3
@@ -1302,7 +1302,7 @@ def github(
     with open(github_output, "a", encoding="utf-8") as wfh:
         wfh.write(f"release-messsage-file={release_message_path.resolve()}\n")
 
-    releases = _get_salt_releases(ctx, repository)
+    releases = get_salt_releases(ctx, repository)
     if Version(salt_version) >= releases[-1]:
         make_latest = True
     else:
@@ -1343,7 +1343,7 @@ def confirm_unreleased(
     """
     Confirm that the passed version is not yet tagged and/or released.
     """
-    releases = _get_salt_releases(ctx, repository)
+    releases = get_salt_releases(ctx, repository)
     if Version(salt_version) in releases:
         ctx.error(f"There's already a '{salt_version}' tag or github release.")
         ctx.exit(1)
@@ -1402,57 +1402,6 @@ def confirm_staged(ctx: Context, salt_version: str, repository: str = "saltstack
     ctx.exit(0)
 
 
-def _get_salt_releases(ctx: Context, repository: str) -> list[Version]:
-    """
-    Return a list of salt versions
-    """
-    versions = set()
-    with ctx.web as web:
-        headers = {
-            "Accept": "application/vnd.github+json",
-        }
-        if "GITHUB_TOKEN" in os.environ:
-            headers["Authorization"] = f"Bearer {os.environ['GITHUB_TOKEN']}"
-        web.headers.update(headers)
-        ret = web.get(f"https://api.github.com/repos/{repository}/tags")
-        if ret.status_code != 200:
-            ctx.error(
-                f"Failed to get the tags for repository {repository!r}: {ret.reason}"
-            )
-            ctx.exit(1)
-        for tag in ret.json():
-            name = tag["name"]
-            if name.startswith("v"):
-                name = name[1:]
-            if "-" in name:
-                # We're not going to parse dash tags
-                continue
-            if "docs" in name:
-                # We're not going to consider doc tags
-                continue
-            versions.add(Version(name))
-
-        # Now let's go through the github releases
-        ret = web.get(f"https://api.github.com/repos/{repository}/releases")
-        if ret.status_code != 200:
-            ctx.error(
-                f"Failed to get the releases for repository {repository!r}: {ret.reason}"
-            )
-            ctx.exit(1)
-        for release in ret.json():
-            name = release["name"]
-            if name.startswith("v"):
-                name = name[1:]
-            if name and "-" not in name and "docs" not in name:
-                # We're not going to parse dash or docs releases
-                versions.add(Version(name))
-            name = release["tag_name"]
-            if "-" not in name and "docs" not in name:
-                # We're not going to parse dash or docs releases
-                versions.add(Version(name))
-    return sorted(versions)
-
-
 def _get_repo_detailed_file_list(
     bucket_name: str,
     bucket_folder: str = "",

tools/utils.py

@@ -118,3 +118,54 @@ class Version(packaging.version.Version):
 
     def __hash__(self):
         return hash(str(self))
+
+
+def get_salt_releases(ctx: Context, repository: str) -> list[Version]:
+    """
+    Return a list of salt versions
+    """
+    versions = set()
+    with ctx.web as web:
+        headers = {
+            "Accept": "application/vnd.github+json",
+        }
+        if "GITHUB_TOKEN" in os.environ:
+            headers["Authorization"] = f"Bearer {os.environ['GITHUB_TOKEN']}"
+        web.headers.update(headers)
+        ret = web.get(f"https://api.github.com/repos/{repository}/tags")
+        if ret.status_code != 200:
+            ctx.error(
+                f"Failed to get the tags for repository {repository!r}: {ret.reason}"
+            )
+            ctx.exit(1)
+        for tag in ret.json():
+            name = tag["name"]
+            if name.startswith("v"):
+                name = name[1:]
+            if "-" in name:
+                # We're not going to parse dash tags
+                continue
+            if "docs" in name:
+                # We're not going to consider doc tags
+                continue
+            versions.add(Version(name))
+
+        # Now let's go through the github releases
+        ret = web.get(f"https://api.github.com/repos/{repository}/releases")
+        if ret.status_code != 200:
+            ctx.error(
+                f"Failed to get the releases for repository {repository!r}: {ret.reason}"
+            )
+            ctx.exit(1)
+        for release in ret.json():
+            name = release["name"]
+            if name.startswith("v"):
+                name = name[1:]
+            if name and "-" not in name and "docs" not in name:
+                # We're not going to parse dash or docs releases
+                versions.add(Version(name))
+            name = release["tag_name"]
+            if "-" not in name and "docs" not in name:
+                # We're not going to parse dash or docs releases
+                versions.add(Version(name))
+    return sorted(versions)