package build matrix wip

.github/actionlint.yaml

@@ -1,17 +1,5 @@
 self-hosted-runner:
   # Labels of self-hosted runner in array of string
   labels:
-    - bastion
-    - x86_64
-    - arm64
-    - aarch64
-    - amd64
-    - repo-nightly
-    - repo-staging
-    - repo-release
-    - medium
-    - large
-    - macos-13-arm64
-    - macos-13-xlarge
     - linux-x86_64
     - linux-arm64

.github/workflows/ci.yml

@@ -462,6 +462,22 @@ jobs:
       cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.15
       nox-archive-hash: "${{ needs.prepare-workflow.outputs.nox-archive-hash }}"
       matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
+  test-packages:
+    name: Package Test
+    needs:
+      - prepare-workflow
+      - build-pkgs-onedir
+      - build-ci-deps
+    uses: ./.github/workflows/test-packages-action.yml
+    with:
+      nox-session: ci-test-onedir
+      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      nox-version: 2022.8.7
+      python-version: "3.10"
+      cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.15
+      skip-code-coverage: ${{ fromJSON(needs.prepare-workflow.outputs.testrun)['skip_code_coverage'] }}
+      testing-releases: ${{ needs.prepare-workflow.outputs.testing-releases }}
+      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['pkg-test-matrix']) }}
 
   test-pkg-rockylinux-8:
     name: Rocky Linux 8 Package Test

.github/workflows/nightly.yml

@@ -539,6 +539,22 @@ jobs:
       cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.15
       nox-archive-hash: "${{ needs.prepare-workflow.outputs.nox-archive-hash }}"
       matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
+  test-packages:
+    name: Package Test
+    needs:
+      - prepare-workflow
+      - build-pkgs-onedir
+      - build-ci-deps
+    uses: ./.github/workflows/test-packages-action.yml
+    with:
+      nox-session: ci-test-onedir
+      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      nox-version: 2022.8.7
+      python-version: "3.10"
+      cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.15
+      skip-code-coverage: true
+      testing-releases: ${{ needs.prepare-workflow.outputs.testing-releases }}
+      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['pkg-test-matrix']) }}
 
   test-pkg-rockylinux-8:
     name: Rocky Linux 8 Package Test

.github/workflows/scheduled.yml

@@ -501,6 +501,22 @@ jobs:
       cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.15
       nox-archive-hash: "${{ needs.prepare-workflow.outputs.nox-archive-hash }}"
       matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
+  test-packages:
+    name: Package Test
+    needs:
+      - prepare-workflow
+      - build-pkgs-onedir
+      - build-ci-deps
+    uses: ./.github/workflows/test-packages-action.yml
+    with:
+      nox-session: ci-test-onedir
+      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      nox-version: 2022.8.7
+      python-version: "3.10"
+      cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.15
+      skip-code-coverage: true
+      testing-releases: ${{ needs.prepare-workflow.outputs.testing-releases }}
+      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['pkg-test-matrix']) }}
 
   test-pkg-rockylinux-8:
     name: Rocky Linux 8 Package Test

.github/workflows/staging.yml

@@ -525,6 +525,22 @@ jobs:
       cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.15
       nox-archive-hash: "${{ needs.prepare-workflow.outputs.nox-archive-hash }}"
       matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
+  test-packages:
+    name: Package Test
+    needs:
+      - prepare-workflow
+      - build-pkgs-onedir
+      - build-ci-deps
+    uses: ./.github/workflows/test-packages-action.yml
+    with:
+      nox-session: ci-test-onedir
+      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      nox-version: 2022.8.7
+      python-version: "3.10"
+      cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.15
+      skip-code-coverage: true
+      testing-releases: ${{ needs.prepare-workflow.outputs.testing-releases }}
+      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['pkg-test-matrix']) }}
 
   test-pkg-rockylinux-8:
     name: Rocky Linux 8 Package Test

.github/workflows/templates/test-salt-pkg.yml.jinja

@@ -1,3 +1,21 @@
+  <%- set job_name = "test-packages" %>
+  <{ job_name }>:
+    name: Package Test
+    needs:
+      - prepare-workflow
+      - build-pkgs-onedir
+      - build-ci-deps
+    uses: ./.github/workflows/test-packages-action.yml
+    with:
+      nox-session: ci-test-onedir
+      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
+      nox-version: <{ nox_version  }>
+      python-version: "<{ gh_actions_workflows_python_version }>"
+      cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }>
+      skip-code-coverage: <{ skip_test_coverage_check }>
+      testing-releases: ${{ needs.prepare-workflow.outputs.testing-releases }}
+      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['pkg-test-matrix']) }}
+
   <%- for os in test_salt_pkg_listing["linux"] %>
     <%- set job_name = os.job_name %>
 

.github/workflows/test-packages-action.yml

@@ -0,0 +1,239 @@
+name: Test Artifact
+
+on:
+  workflow_call:
+    inputs:
+      salt-version:
+        type: string
+        required: true
+        description: The Salt version of the packages to install and test
+      cache-prefix:
+        required: true
+        type: string
+        description: Seed used to invalidate caches
+      testing-releases:
+        required: true
+        type: string
+        description: A JSON list of releases to test upgrades against
+      nox-version:
+        required: true
+        type: string
+        description: The nox version to install
+      python-version:
+        required: false
+        type: string
+        description: The python version to run tests with
+        default: "3.10"
+      nox-session:
+        required: false
+        type: string
+        description: The nox session to run
+        default: ci-test-onedir
+      skip-code-coverage:
+        required: false
+        type: boolean
+        description: Skip code coverage
+        default: false
+      package-name:
+        required: false
+        type: string
+        description: The onedir package name to use
+        default: salt
+      matrix:
+        required: true
+        type: string
+        description: Json job matrix config
+
+env:
+  COLUMNS: 190
+  AWS_MAX_ATTEMPTS: "10"
+  AWS_RETRY_MODE: "adaptive"
+  PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
+  PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
+  PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
+  PIP_DISABLE_PIP_VERSION_CHECK: "1"
+  RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
+  USE_S3_CACHE: 'false'
+
+jobs:
+
+  test:
+    name: Test
+    runs-on:
+      - ${{ matrix.arch == 'x86_64' && 'ubuntu-24.04' || 'linux-arm64' }}
+    container:
+      image: ${{ matrix.container }}
+      options: --privileged
+    timeout-minutes: 120  # 2 Hours - More than this and something is wrong
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(inputs.matrix)['linux'] }}
+
+    steps:
+
+      - name: "Throttle Builds"
+        shell: bash
+        run: |
+          t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t"
+
+      - name: "Set `TIMESTAMP` environment variable"
+        shell: bash
+        run: |
+          echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV"
+
+      - name: Checkout Source Code
+        uses: actions/checkout@v4
+
+      - name: Download Packages
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-${{ matrix.arch }}-${{ matrix.pkg_type }}
+          path: artifacts/pkg/
+
+      - name: Download Onedir Tarball as an Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ matrix.platform }}-${{ matrix.arch }}.tar.xz
+          path: artifacts/
+
+      - name: Decompress Onedir Tarball
+        shell: bash
+        run: |
+          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
+          cd artifacts
+          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ matrix.platform }}-${{ matrix.arch }}.tar.xz
+
+      #- name: Set up Python ${{ inputs.python-version }}
+      #  uses: actions/setup-python@v5
+      #  with:
+      #    python-version: "${{ inputs.python-version }}"
+
+      - name: Install Nox
+        run: |
+          python3 -m pip install 'nox==${{ inputs.nox-version }}'
+        env:
+          PIP_INDEX_URL: https://pypi.org/simple
+
+      - name: List Packages
+        run: |
+          tree artifacts/pkg/
+
+      - name: Download nox.linux.${{ matrix.arch }}.tar.* artifact for session ${{ inputs.nox-session }}
+        uses: actions/download-artifact@v4
+        with:
+          name: nox-linux-${{ matrix.arch }}-${{ inputs.nox-session }}
+
+      - name: Decompress .nox Directory
+        run: |
+          nox --force-color -e decompress-dependencies -- linux ${{ matrix.arch }}
+
+      - name: Setup Python Tools Scripts
+        uses: ./.github/actions/setup-python-tools-scripts
+        with:
+          cache-prefix: ${{ inputs.cache-prefix }}
+
+      - name: List Free Space
+        run: |
+          df -h || true
+
+#      - name: check systemd
+#        run: systemctl
+#
+      #- name: Mock systemd
+      #  run: |
+      #    wget https://raw.githubusercontent.com/gdraheim/docker-systemctl-replacement/refs/heads/master/files/docker/systemctl3.py
+      #    cp systemctl3.py /usr/bin/systemctl
+      #    chmod +x /usr/bin/systemctl
+      #
+      #- name: install pkg test
+      #  run: |
+      #    dpkg -i ./artifacts/pkg/salt-common_${{ inputs.salt-version}}_${{ inputs.arch == 'x86_64' && 'amd64' || 'arm64' }}.deb ./artifacts/pkg/salt-master_${{ inputs.salt-version}}_${{ inputs.arch == 'x86_64' && 'amd64' || 'arm64' }}.deb
+
+      - name: Show System Info
+        env:
+          SKIP_REQUIREMENTS_INSTALL: "1"
+          PRINT_SYSTEM_INFO_ONLY: "1"
+        run: |
+          nox --force-color -e ${{ inputs.nox-session }}-pkgs -- ${{ matrix.tests-chunk }}
+
+      - name: Run Package Tests
+        env:
+          SKIP_REQUIREMENTS_INSTALL: "1"
+          RERUN_FAILURES: "1"
+          GITHUB_ACTIONS_PIPELINE: "1"
+          SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
+          COVERAGE_CONTEXT: ${{ matrix.distro-slug }}
+        run: |
+          nox --force-color -e ${{ inputs.nox-session }}-pkgs -- ${{ matrix.tests-chunk }} \
+          ${{ matrix.version && format('--prev-version={0}', matrix.version) || ''}}
+
+      - name: Upload Test Run Log Artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: pkg-testrun-log-artifacts-${{ matrix.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ env.TIMESTAMP }}
+          path: |
+            artifacts/logs
+          include-hidden-files: true
+
+      - name: Upload Test Run Artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: pkg-testrun-artifacts-${{ matrix.distro-slug }}${{ matrix.fips && '-fips' || '' }}-${{ matrix.pkg_type }}-${{ matrix.arch }}-${{ matrix.tests-chunk }}-${{ matrix.version || 'no-version'}}-${{ env.TIMESTAMP }}
+          path: |
+            artifacts/
+            !artifacts/pkg/*
+            !artifacts/salt/*
+            !artifacts/salt-*.tar.*
+          include-hidden-files: true
+
+  report:
+    name: Report
+    runs-on: ubuntu-latest
+    if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && needs.test.result != 'cancelled' && needs.test.result != 'skipped'
+    needs:
+      - test
+    strategy:
+      matrix:
+        include: ${{ fromJSON(inputs.matrix)['linux'] }}
+
+    steps:
+      - name: Checkout Source Code
+        uses: actions/checkout@v4
+
+      - name: "Throttle Builds"
+        shell: bash
+        run: |
+          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"
+
+      - name: Wait For Artifacts
+        run: |
+          sleep 60
+
+      - name: Merge Test Run Artifacts
+        continue-on-error: true
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: pkg-testrun-artifacts-${{ matrix.distro-slug }}${{ matrix.fips && '-fips' || '' }}-${{ matrix.pkg_type }}
+          pattern: pkg-testrun-artifacts-${{ matrix.distro-slug }}${{ matrix.fips && '-fips' || '' }}-${{ matrix.pkg_type }}-*
+          separate-directories: true
+          delete-merged: true
+
+      - name: Wait For Artifacts 2
+        run: |
+          sleep 60
+
+      - name: Download Test Run Artifacts
+        id: download-test-run-artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts/
+          pattern: pkg-testrun-artifacts-${{ matrix.distro-slug }}${{ matrix.fips && '-fips' || '' }}-${{ matrix.pkg_type }}*
+          merge-multiple: true
+
+      - name: Show Test Run Artifacts
+        if: always()
+        run: |
+          tree -a artifacts

tools/ci.py

@@ -9,6 +9,7 @@ import json
 import logging
 import os
 import pathlib
+import pprint
 import random
 import shutil
 import sys
@@ -1609,19 +1610,47 @@ def workflow_config(
     str_releases = [str(version) for version in testing_releases]
     ctx.info(f"str_releases {str_releases}")
 
-    pkg_matrix = [
-        dict(
-            {
-                "tests-chunk": "install",
-                "version": None,
-            },
-            **_.as_dict(),
-        )
-        for _ in TEST_SALT_PKG_LISTING["linux"]  # type: ignore
-    ]
-    import pprint
+    platforms = ["linux", "macos", "windows"]
+    pkg_test_matrix = {}
+    for platform in platforms:
+        pkg_test_matrix[platform] = [
+            dict(
+                {
+                    "tests-chunk": "install",
+                    "version": None,
+                },
+                **_.as_dict(),
+            )
+            for _ in TEST_SALT_PKG_LISTING[platform]  # type: ignore
+        ]
+    for version in str_releases:
+        for platform in platforms:
+            pkg_test_matrix[platform] += [
+                dict(
+                    {
+                        "tests-chunk": "upgrade",
+                        "version": version,
+                    },
+                    **_.as_dict(),
+                )
+                for _ in TEST_SALT_PKG_LISTING[platform]  # type: ignore
+            ]
+            pkg_test_matrix[platform] += [
+                dict(
+                    {
+                        "tests-chunk": "downgrade",
+                        "version": version,
+                    },
+                    **_.as_dict(),
+                )
+                for _ in TEST_SALT_PKG_LISTING[platform]  # type: ignore
+            ]
 
-    ctx.info(f"{pprint.pformat(pkg_matrix)}")
+    ctx.info(f"{'==== pkg test matrix ====':^80s}")
+    ctx.info(f"{pprint.pformat(pkg_test_matrix)}")
+    ctx.info(f"{'==== end pkg test matrix ====':^80s}")
+
+    config["pkg-test-matrix"] = pkg_test_matrix  # type: ignore
 
     ctx.info("Jobs selected are")
     for x, y in jobs.items():

tools/utils/__init__.py

@@ -118,6 +118,7 @@ class MacOS(OS):
             "job_name": self.job_name,
         }
 
+
 @attr.s(frozen=True, slots=True)
 class MacOSPkg(MacOS):