use build deps matrix from config

.github/workflows/build-deps-onedir.yml

@@ -20,10 +20,10 @@ on:
         required: true
         type: string
         description: The version of python to use with relenv
-      kind:
-        type: string
+      matrix:
         required: true
-        description: The kind of tests to run windows, macos, or linux
+        type: string
+        description: Json job matrix config
 
 env:
   RELENV_DATA: "${{ github.workspace }}/.relenv"
@@ -37,47 +37,14 @@ env:
 
 jobs:
 
-  generate-matrix:
-    name: Test Matrix (${{ inputs.kind }})
-    runs-on: ubuntu-latest
-    outputs:
-      matrix-include: ${{ steps.generate-matrix.outputs.matrix }}
-    steps:
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.10.14"
-
-      - name: "Throttle Builds"
-        shell: bash
-        run: |
-          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"
-
-      - name: Checkout Source Code
-        uses: actions/checkout@v4
-
-      - name: Setup Python Tools Scripts
-        uses: ./.github/actions/setup-python-tools-scripts
-        with:
-          cache-prefix: ${{ inputs.cache-seed }}
-        env:
-          PIP_INDEX_URL: https://pypi.org/simple
-
-      - name: Generate Test Matrix
-        id: generate-matrix
-        run: tools ci build-matrix ${{ inputs.kind }}
-
-
   build-deps-linux:
     name: Linux
-    if: ${{ inputs.kind  == 'linux' }}
     runs-on:
       - ${{ matrix.arch == 'x86_64' && 'ubuntu-latest' || 'linux-arm64' }}
-    needs:
-      - generate-matrix
     strategy:
       fail-fast: false
       matrix:
-        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include) }}
+        include: ${{ fromJSON(inputs.matrix)['linux'] }}
     env:
       USE_S3_CACHE: 'false'
     steps:
@@ -114,14 +81,11 @@ jobs:
 
   build-deps-macos:
     name: macOS
-    if: ${{ inputs.kind  == 'macOS' }}
     strategy:
       fail-fast: false
       max-parallel: 2
       matrix:
-        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include) }}
-    needs:
-      - generate-matrix
+        include: ${{ fromJSON(inputs.matrix)['macos'] }}
     runs-on:
       - ${{ matrix.arch == 'arm64' && 'macos-14' || 'macos-13' }}
     env:
@@ -170,16 +134,11 @@ jobs:
 
   build-deps-windows:
     name: Windows
-    if: ${{ inputs.kind  == 'windows' }}
     strategy:
       fail-fast: false
       max-parallel: 2
       matrix:
-        arch:
-          - x86
-          - amd64
-    needs:
-      - generate-matrix
+        include: ${{ fromJSON(inputs.matrix)['windows'] }}
     runs-on: windows-latest
     env:
       USE_S3_CACHE: 'false'

.github/workflows/ci.yml

@@ -404,9 +404,9 @@ jobs:
         with:
           salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
 
-  build-deps-onedir-linux:
+  build-deps-onedir:
     name: Build Onedir Dependencies
-    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir-linux'] }}
+    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir'] }}
     needs:
       - prepare-workflow
     uses: ./.github/workflows/build-deps-onedir.yml
@@ -415,40 +415,14 @@ jobs:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
       relenv-version: "0.18.0"
       python-version: "3.10.15"
-      kind: linux
-
-  build-deps-onedir-macos:
-    name: Build Onedir Dependencies
-    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir-macos'] }}
-    needs:
-      - prepare-workflow
-    uses: ./.github/workflows/build-deps-onedir.yml
-    with:
-      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
-      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
-      relenv-version: "0.18.0"
-      python-version: "3.10.15"
-      kind: macos
-
-  build-deps-onedir-windows:
-    name: Build Onedir Dependencies
-    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir-windows'] }}
-    needs:
-      - prepare-workflow
-    uses: ./.github/workflows/build-deps-onedir.yml
-    with:
-      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
-      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
-      relenv-version: "0.18.0"
-      python-version: "3.10.15"
-      kind: windows
+      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
 
   build-salt-onedir-linux:
     name: Build Salt Onedir
     if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-salt-onedir-linux'] }}
     needs:
       - prepare-workflow
-      - build-deps-onedir-linux
+      - build-deps-onedir
       - build-source-tarball
     uses: ./.github/workflows/build-salt-onedir.yml
     with:
@@ -463,7 +437,7 @@ jobs:
     if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-salt-onedir-macos'] }}
     needs:
       - prepare-workflow
-      - build-deps-onedir-macos
+      - build-deps-onedir
       - build-source-tarball
     uses: ./.github/workflows/build-salt-onedir.yml
     with:
@@ -478,7 +452,7 @@ jobs:
     if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-salt-onedir-windows'] }}
     needs:
       - prepare-workflow
-      - build-deps-onedir-windows
+      - build-deps-onedir
       - build-source-tarball
     uses: ./.github/workflows/build-salt-onedir.yml
     with:
@@ -2232,9 +2206,7 @@ jobs:
       - lint
       - nsis-tests
       - build-docs
-      - build-deps-onedir-linux
-      - build-deps-onedir-macos
-      - build-deps-onedir-windows
+      - build-deps-onedir
       - build-salt-onedir-linux
       - build-salt-onedir-macos
       - build-salt-onedir-windows

.github/workflows/nightly.yml

@@ -458,9 +458,9 @@ jobs:
         with:
           salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
 
-  build-deps-onedir-linux:
+  build-deps-onedir:
     name: Build Onedir Dependencies
-    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir-linux'] }}
+    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir'] }}
     needs:
       - prepare-workflow
     uses: ./.github/workflows/build-deps-onedir.yml
@@ -469,40 +469,14 @@ jobs:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
       relenv-version: "0.18.0"
       python-version: "3.10.15"
-      kind: linux
-
-  build-deps-onedir-macos:
-    name: Build Onedir Dependencies
-    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir-macos'] }}
-    needs:
-      - prepare-workflow
-    uses: ./.github/workflows/build-deps-onedir.yml
-    with:
-      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
-      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
-      relenv-version: "0.18.0"
-      python-version: "3.10.15"
-      kind: macos
-
-  build-deps-onedir-windows:
-    name: Build Onedir Dependencies
-    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir-windows'] }}
-    needs:
-      - prepare-workflow
-    uses: ./.github/workflows/build-deps-onedir.yml
-    with:
-      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
-      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
-      relenv-version: "0.18.0"
-      python-version: "3.10.15"
-      kind: windows
+      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
 
   build-salt-onedir-linux:
     name: Build Salt Onedir
     if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-salt-onedir-linux'] }}
     needs:
       - prepare-workflow
-      - build-deps-onedir-linux
+      - build-deps-onedir
       - build-source-tarball
     uses: ./.github/workflows/build-salt-onedir.yml
     with:
@@ -517,7 +491,7 @@ jobs:
     if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-salt-onedir-macos'] }}
     needs:
       - prepare-workflow
-      - build-deps-onedir-macos
+      - build-deps-onedir
       - build-source-tarball
     uses: ./.github/workflows/build-salt-onedir.yml
     with:
@@ -532,7 +506,7 @@ jobs:
     if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-salt-onedir-windows'] }}
     needs:
       - prepare-workflow
-      - build-deps-onedir-windows
+      - build-deps-onedir
       - build-source-tarball
     uses: ./.github/workflows/build-salt-onedir.yml
     with:
@@ -3014,9 +2988,7 @@ jobs:
       - lint
       - nsis-tests
       - build-docs
-      - build-deps-onedir-linux
-      - build-deps-onedir-macos
-      - build-deps-onedir-windows
+      - build-deps-onedir
       - build-salt-onedir-linux
       - build-salt-onedir-macos
       - build-salt-onedir-windows

.github/workflows/scheduled.yml

@@ -443,9 +443,9 @@ jobs:
         with:
           salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
 
-  build-deps-onedir-linux:
+  build-deps-onedir:
     name: Build Onedir Dependencies
-    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir-linux'] }}
+    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir'] }}
     needs:
       - prepare-workflow
     uses: ./.github/workflows/build-deps-onedir.yml
@@ -454,40 +454,14 @@ jobs:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
       relenv-version: "0.18.0"
       python-version: "3.10.15"
-      kind: linux
-
-  build-deps-onedir-macos:
-    name: Build Onedir Dependencies
-    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir-macos'] }}
-    needs:
-      - prepare-workflow
-    uses: ./.github/workflows/build-deps-onedir.yml
-    with:
-      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
-      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
-      relenv-version: "0.18.0"
-      python-version: "3.10.15"
-      kind: macos
-
-  build-deps-onedir-windows:
-    name: Build Onedir Dependencies
-    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir-windows'] }}
-    needs:
-      - prepare-workflow
-    uses: ./.github/workflows/build-deps-onedir.yml
-    with:
-      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
-      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
-      relenv-version: "0.18.0"
-      python-version: "3.10.15"
-      kind: windows
+      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
 
   build-salt-onedir-linux:
     name: Build Salt Onedir
     if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-salt-onedir-linux'] }}
     needs:
       - prepare-workflow
-      - build-deps-onedir-linux
+      - build-deps-onedir
       - build-source-tarball
     uses: ./.github/workflows/build-salt-onedir.yml
     with:
@@ -502,7 +476,7 @@ jobs:
     if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-salt-onedir-macos'] }}
     needs:
       - prepare-workflow
-      - build-deps-onedir-macos
+      - build-deps-onedir
       - build-source-tarball
     uses: ./.github/workflows/build-salt-onedir.yml
     with:
@@ -517,7 +491,7 @@ jobs:
     if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-salt-onedir-windows'] }}
     needs:
       - prepare-workflow
-      - build-deps-onedir-windows
+      - build-deps-onedir
       - build-source-tarball
     uses: ./.github/workflows/build-salt-onedir.yml
     with:
@@ -2109,9 +2083,7 @@ jobs:
       - lint
       - nsis-tests
       - build-docs
-      - build-deps-onedir-linux
-      - build-deps-onedir-macos
-      - build-deps-onedir-windows
+      - build-deps-onedir
       - build-salt-onedir-linux
       - build-salt-onedir-macos
       - build-salt-onedir-windows

.github/workflows/staging.yml

@@ -440,9 +440,9 @@ jobs:
         with:
           salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
 
-  build-deps-onedir-linux:
+  build-deps-onedir:
     name: Build Onedir Dependencies
-    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir-linux'] }}
+    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir'] }}
     needs:
       - prepare-workflow
     uses: ./.github/workflows/build-deps-onedir.yml
@@ -451,40 +451,14 @@ jobs:
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
       relenv-version: "0.18.0"
       python-version: "3.10.15"
-      kind: linux
-
-  build-deps-onedir-macos:
-    name: Build Onedir Dependencies
-    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir-macos'] }}
-    needs:
-      - prepare-workflow
-    uses: ./.github/workflows/build-deps-onedir.yml
-    with:
-      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
-      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
-      relenv-version: "0.18.0"
-      python-version: "3.10.15"
-      kind: macos
-
-  build-deps-onedir-windows:
-    name: Build Onedir Dependencies
-    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-deps-onedir-windows'] }}
-    needs:
-      - prepare-workflow
-    uses: ./.github/workflows/build-deps-onedir.yml
-    with:
-      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
-      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
-      relenv-version: "0.18.0"
-      python-version: "3.10.15"
-      kind: windows
+      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
 
   build-salt-onedir-linux:
     name: Build Salt Onedir
     if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-salt-onedir-linux'] }}
     needs:
       - prepare-workflow
-      - build-deps-onedir-linux
+      - build-deps-onedir
       - build-source-tarball
     uses: ./.github/workflows/build-salt-onedir.yml
     with:
@@ -499,7 +473,7 @@ jobs:
     if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-salt-onedir-macos'] }}
     needs:
       - prepare-workflow
-      - build-deps-onedir-macos
+      - build-deps-onedir
       - build-source-tarball
     uses: ./.github/workflows/build-salt-onedir.yml
     with:
@@ -514,7 +488,7 @@ jobs:
     if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['build-salt-onedir-windows'] }}
     needs:
       - prepare-workflow
-      - build-deps-onedir-windows
+      - build-deps-onedir
       - build-source-tarball
     uses: ./.github/workflows/build-salt-onedir.yml
     with:
@@ -3167,9 +3141,7 @@ jobs:
       - lint
       - nsis-tests
       - build-docs
-      - build-deps-onedir-linux
-      - build-deps-onedir-macos
-      - build-deps-onedir-windows
+      - build-deps-onedir
       - build-salt-onedir-linux
       - build-salt-onedir-macos
       - build-salt-onedir-windows

.github/workflows/templates/ci.yml.jinja

@@ -243,7 +243,7 @@
   <%- endif %>
 
 
-  <%- set job_name = "build-deps-onedir-linux" %>
+  <%- set job_name = "build-deps-onedir" %>
   <%- if includes.get(job_name, True) %>
 
   <{ job_name }>:
@@ -258,49 +258,10 @@
       salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
       relenv-version: "<{ relenv_version }>"
       python-version: "<{ python_version }>"
-      kind: linux
+      matrix: ${{ toJSON(fromJSON(needs.prepare-workflow.outputs.config)['build-matrix']) }}
 
   <%- endif %>
 
-  <%- set job_name = "build-deps-onedir-macos" %>
-  <%- if includes.get(job_name, True) %>
-
-  <{ job_name }>:
-    <%- do conclusion_needs.append(job_name) %>
-    name: Build Onedir Dependencies
-    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['<{ job_name }>'] }}
-    needs:
-      - prepare-workflow
-    uses: ./.github/workflows/build-deps-onedir.yml
-    with:
-      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
-      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
-      relenv-version: "<{ relenv_version }>"
-      python-version: "<{ python_version }>"
-      kind: macos
-
-  <%- endif %>
-
-  <%- set job_name = "build-deps-onedir-windows" %>
-  <%- if includes.get(job_name, True) %>
-
-  <{ job_name }>:
-    <%- do conclusion_needs.append(job_name) %>
-    name: Build Onedir Dependencies
-    if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['<{ job_name }>'] }}
-    needs:
-      - prepare-workflow
-    uses: ./.github/workflows/build-deps-onedir.yml
-    with:
-      cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }}
-      salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
-      relenv-version: "<{ relenv_version }>"
-      python-version: "<{ python_version }>"
-      kind: windows
-
-  <%- endif %>
-
-
   <%- set job_name = "build-salt-onedir-linux" %>
   <%- if includes.get(job_name, True) %>
 
@@ -310,7 +271,7 @@
     if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['<{ job_name }>'] }}
     needs:
       - prepare-workflow
-      - build-deps-onedir-linux
+      - build-deps-onedir
       - build-source-tarball
     uses: ./.github/workflows/build-salt-onedir.yml
     with:
@@ -330,7 +291,7 @@
     if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['<{ job_name }>'] }}
     needs:
       - prepare-workflow
-      - build-deps-onedir-macos
+      - build-deps-onedir
       - build-source-tarball
     uses: ./.github/workflows/build-salt-onedir.yml
     with:
@@ -350,7 +311,7 @@
     if: ${{ fromJSON(needs.prepare-workflow.outputs.config)['jobs']['<{ job_name }>'] }}
     needs:
       - prepare-workflow
-      - build-deps-onedir-windows
+      - build-deps-onedir
       - build-source-tarball
     uses: ./.github/workflows/build-salt-onedir.yml
     with:

tools/ci.py

@@ -576,8 +576,23 @@ def build_matrix(
     github_output = os.environ.get("GITHUB_OUTPUT")
     if github_output is None:
         ctx.warn("The 'GITHUB_OUTPUT' variable is not set.")
+    _matrix = _build_matrix(kind)
+    if github_output is not None:
+        with open(github_output, "a", encoding="utf-8") as wfh:
+            wfh.write(f"matrix={json.dumps(_matrix)}\n")
+    else:
+        ctx.warn("The 'GITHUB_OUTPUT' variable is not set.")
+    ctx.exit(0)
+
+
+def _build_matrix(kind):
     _matrix = [{"arch": "x86_64"}]
-    if kind == "macos":
+    if kind == "windows":
+        _matrix = [
+            {"arch": "amd64"},
+            {"arch": "x86"},
+        ]
+    elif kind == "macos":
         _matrix.append({"arch": "arm64"})
     elif (
         kind == "linux"
@@ -585,12 +600,7 @@ def build_matrix(
         and os.environ["LINUX_ARM_RUNNER"] != "0"
     ):
         _matrix.append({"arch": "arm64"})
-    if github_output is not None:
-        with open(github_output, "a", encoding="utf-8") as wfh:
-            wfh.write(f"matrix={json.dumps(_matrix)}\n")
-    else:
-        ctx.warn("The 'GITHUB_OUTPUT' variable is not set.")
-    ctx.exit(0)
+    return _matrix
 
 
 @ci.command(
@@ -1599,6 +1609,9 @@ def workflow_config(
         jobs["test-pkg-download"] = False
 
     config["jobs"] = jobs
+    config["build-matrix"] = {
+        kind: _build_matrix(kind) for kind in ["linux", "macos", "windows"]  # type: ignore
+    }
     ctx.info("Jobs selected are")
     for x, y in jobs.items():
         ctx.info(f"{x} = {y}")