---
name: Test Artifact

on:
  workflow_call:
    inputs:
      nox-session:
        required: true
        type: string
        description: The nox session to run
      testrun:
        required: true
        type: string
        description: JSON string containing information about what and how to run the test suite
      python-version:
        required: false
        type: string
        description: The python version to run tests with
        default: "3.10"
      salt-version:
        type: string
        required: true
        description: The Salt version to set prior to running tests.
      cache-prefix:
        required: true
        type: string
        description: Seed used to invalidate caches
      nox-version:
        required: true
        type: string
        description: The nox version to install
      package-name:
        required: false
        type: string
        description: The onedir package name to use
        default: salt
      skip-code-coverage:
        required: false
        type: boolean
        description: Skip code coverage
        default: false
      workflow-slug:
        required: false
        type: string
        description: Which workflow is running.
        default: ci
      default-timeout:
        required: false
        type: number
        description: Timeout, in minutes, for the test job(Default 360, 6 hours).
        default: 360
      matrix:
        required: true
        type: string
        description: Json job matrix config
      linux_arm_runner:
        required: true
        type: string
        description: Json job matrix config

env:
  COLUMNS: 190
  PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
  PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
  PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
  PIP_DISABLE_PIP_VERSION_CHECK: "1"
  RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"

jobs:

  test-linux:
    name: ${{ matrix.display_name }} ${{ matrix.tests-chunk }} ${{ matrix.transport }}${{ matrix.fips && '(fips)' || '' }}${{ matrix.test-group && ' ' || '' }}${{ matrix.test-group && matrix.test-group || '' }}
    runs-on: ${{ matrix.arch == 'x86_64' && 'ubuntu-24.04' || inputs.linux_arm_runner }}
    if: ${{ !cancelled() && toJSON(fromJSON(inputs.matrix)['linux-x86_64']) != '[]' }}
    # Full test runs. Each chunk should never take more than 2 hours.
    # Partial test runs(no chunk parallelization), 6 Hours
    timeout-minutes: ${{ fromJSON(inputs.testrun)['type'] == 'full' && inputs.default-timeout || 360 }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(inputs.matrix)['linux-x86_64'] }}
    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t"

      - name: "Set `TIMESTAMP` environment variable"
        shell: bash
        run: |
          echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Setup Python ${{ inputs.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: "${{ inputs.python-version }}"

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-prefix }}

      - name: Free Disk Space Before Build
        run: |
          echo "Disk space before cleanup:"
          df -h
          sudo rm -rf /usr/local/.ghcup
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo rm -rf /usr/local/lib/android/sdk/ndk
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/local/share/boost
          sudo apt-get clean
          echo "Disk space after cleanup:"
          df -h

      - name: Setup Salt Version
        run: |
          echo "${{ inputs.salt-version }}" > salt/_version.txt

      - name: Download Onedir Tarball as an Artifact
        uses: actions/download-artifact@v4
        with:
          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ matrix.platform }}-${{ matrix.arch }}.tar.xz
          path: artifacts/

      - name: Decompress Onedir Tarball
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ matrix.platform }}-${{ matrix.arch }}.tar.xz

      - name: "Configure docker"
        run: |
          sudo sed -i '/LimitNOFILE=infinity/c\LimitNOFILE=1048576' /lib/systemd/system/containerd.service
          sudo systemctl daemon-reload
          sudo systemctl restart containerd
          cat /lib/systemd/system/containerd.service

      - name: "Pull container ${{ matrix.container }}"
        run: |
          docker pull ${{ matrix.container }}

      - name: "Create docker network"
        run: |
           docker network create -o "com.docker.network.driver.mtu=1500" --ipv6 --subnet 2001:db8::/64 ip6net

      - name: "Host network config"
        run: |
          ip a

      - name: Free Memory Before Container
        shell: bash
        run: |
          free -h

      - name: "Create container ${{ matrix.container }}"
        env:
          GITHUB_ACTIONS: true
          CI: true
          SKIP_REQUIREMENTS_INSTALL: 1
          PRINT_TEST_SELECTION: 0
          PRINT_TEST_PLAN_ONLY: 0
          PRINT_SYSTEM_INFO: 0
          RERUN_FAILURES: 1
          GITHUB_ACTIONS_PIPELINE: 1
          SKIP_INITIAL_ONEDIR_FAILURES: 1
          SKIP_INITIAL_GH_ACTIONS_FAILURES: 1
          SKIP_CODE_COVERAGE: ${{ inputs.skip-code-coverage && '1' || '0' }}
          CONVERAGE_CONTEXT: ${{ matrix.slug }}
          COLUMNS: 190
          PIP_INDEX_URL: "${{ vars.PIP_INDEX_URL }}"
          PIP_TRUSTED_HOST: "${{ vars.PIP_TRUSTED_HOST }}"
          PIP_EXTRA_INDEX_URL: "${{ vars.PIP_EXTRA_INDEX_URL }}"
          PIP_DISABLE_PIP_VERSION_CHECK: 1
          RAISE_DEPRECATIONS_RUNTIME_ERRORS: 1
          SALT_TRANSPORT: ${{ matrix.transport }}
        run: |
          tools container create ${{ matrix.container }} --name ${{ github.run_id }}_salt-test

      - name: "Start container ${{ matrix.container }}"
        run: |
          /usr/bin/docker start ${{ github.run_id }}_salt-test

      - name: "Show container inspect ${{ matrix.container }}"
        run: |
          /usr/bin/docker inspect ${{ github.run_id }}_salt-test

      - name: Download nox.linux.${{ matrix.arch }}.tar.* artifact for session ${{ inputs.nox-session }}
        uses: actions/download-artifact@v4
        with:
          name: nox-linux-${{ matrix.arch }}-${{ inputs.nox-session }}

      - name: Install Nox
        run: |
          python3 -m pip install 'nox==${{ inputs.nox-version }}'
        env:
          PIP_INDEX_URL: https://pypi.org/simple

      - name: Decompress .nox Directory
        run: |
          docker exec ${{ github.run_id}}_salt-test python3 -m nox --force-color -e decompress-dependencies -- linux ${{ matrix.arch }}

      - name: Download testrun-changed-files.txt
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        uses: actions/download-artifact@v4
        with:
          name: testrun-changed-files.txt

      - name: Current Directory
        run: |
          pwd

      - name: Show System Info
        run: |
          docker exec -e SKIP_REQUIREMENTS_INSTALL=1 -e PRINT_SYSTEM_INFO_ONLY=1 ${{ github.run_id}}_salt-test \
          python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }}

      - name: Free Space on Host
        shell: bash
        run: |
          df -h

      - name: Show container network info
        shell: bash
        run: |
          docker exec ${{ github.run_id}}_salt-test ip addr

      - name: Show container processes
        shell: bash
        run: |
          docker exec ${{ github.run_id}}_salt-test ps auxf

      - name: Free Space on Container
        shell: bash
        run: |
          docker exec ${{ github.run_id}}_salt-test df -h

      - name: Free Memory
        shell: bash
        run: |
          free -h

      - name: Configure apparmor
        run: |
          # Apparmor's unix-chkpwd profile gets in the way of tests needing to
          # authenticate from inside a container.
          cat <<'EOF' | sudo tee /etc/apparmor.d/unix-chkpwd
          abi <abi/4.0>,
          include <tunables/global>
          profile unix-chkpwd /{,usr/}{,s}bin/unix_chkpwd flags=(unconfined) {
            include <abstractions/base>
            include <abstractions/nameservice>
            # To write records to the kernel auditing log.
            capability audit_write,
            network netlink raw,
            /{,usr/}{,s}bin/unix_chkpwd mr,
            /etc/shadow r,
            # systemd userdb, used in nspawn
            /run/host/userdb/*.user r,
            /run/host/userdb/*.user-privileged r,
            # file_inherit
            owner /dev/tty[0-9]* rw,
            include if exists <local/unix-chkpwd>
          }
          EOF
          sudo systemctl restart apparmor
          sudo aa-status

      - name: Run Changed Tests
        id: run-fast-changed-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        run: |
          docker exec ${{ github.run_id}}_salt-test python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            --core-tests --slow-tests --suppress-no-test-exit-code --from-filenames=testrun-changed-files.txt

      - name: Run Fast Tests
        id: run-fast-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }}
        run: |
          docker exec ${{ github.run_id}}_salt-test \
            python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            --suppress-no-test-exit-code

      - name: Run Slow Tests
        id: run-slow-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }}
        run: |
          docker exec ${{ github.run_id}}_salt-test \
            python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            --suppress-no-test-exit-code --no-fast-tests --slow-tests

      - name: Run Core Tests
        id: run-core-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }}
        run: |
          docker exec ${{ github.run_id}}_salt-test \
            python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            --suppress-no-test-exit-code --no-fast-tests --core-tests

      - name: Run Flaky Tests
        id: run-flaky-tests
        if: ${{ fromJSON(inputs.testrun)['selected_tests']['flaky'] }}
        run: |
          docker exec ${{ github.run_id}}_salt-test \
            python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            --suppress-no-test-exit-code --no-fast-tests --flaky-jail

      - name: Run Full Tests
        id: run-full-tests
        if: ${{ fromJSON(inputs.testrun)['type'] == 'full' }}
        run: |
          docker exec ${{ github.run_id}}_salt-test \
            python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            --slow-tests --core-tests --test-group-count=${{ matrix.test-group-count || 1 }} --test-group=${{ matrix.test-group || 1 }}

      - name: Stop Container
        run: |
          docker container stop ${{ github.run_id}}_salt-test

      - name: Remove Container
        run: |
          docker container rm ${{ github.run_id}}_salt-test

      - name: Remove Container Image
        run: |
          docker image rm ${{ matrix.container }}

      - name: Fix file ownership
        run: |
          sudo chown -R "$(id -un)" .

      - name: Combine Coverage Reports
        if: always() && inputs.skip-code-coverage == false
        run: |
          nox --force-color -e combine-coverage

      - name: Prepare Test Run Artifacts
        id: download-artifacts-from-vm
        if: always()
        run: |
          # Delete the salt onedir, we won't need it anymore and it will prevent
          # from it showing in the tree command below
          sudo rm -rf artifacts/salt*
          tree -a artifacts
          if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then
            mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ matrix.slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}
          fi

      - name: Upload Code Coverage Test Run Artifacts
        if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-coverage-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/coverage/
          include-hidden-files: true

      - name: Upload JUnit XML Test Run Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-junit-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}${{ matrix.fips && '(fips)' || '' }}-${{ matrix.tests-chunk }}-${{ matrix.test-group || 1 }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/xml-unittests-output/
          include-hidden-files: true

      - name: Upload Test Run Log Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-log-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}${{ matrix.fips && '(fips)' || '' }}-${{ matrix.tests-chunk }}-${{ matrix.test-group || 1 }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/logs
          include-hidden-files: true

  test-linux-arm64:
    name: ${{ matrix.display_name }} ${{ matrix.tests-chunk }} ${{ matrix.transport }}${{ matrix.fips && '(fips)' || '' }}${{ matrix.test-group && ' ' || '' }}${{ matrix.test-group && matrix.test-group || '' }}
    runs-on: ${{ matrix.arch == 'x86_64' && 'ubuntu-22.04' || inputs.linux_arm_runner }}
    if: ${{ !cancelled() && toJSON(fromJSON(inputs.matrix)['linux-arm64']) != '[]' }}
    # Full test runs. Each chunk should never take more than 2 hours.
    # Partial test runs(no chunk parallelization), 6 Hours
    timeout-minutes: ${{ fromJSON(inputs.testrun)['type'] == 'full' && inputs.default-timeout || 360 }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(inputs.matrix)['linux-arm64'] }}
    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t"

      - name: "Set `TIMESTAMP` environment variable"
        shell: bash
        run: |
          echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV"


      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Setup Python ${{ inputs.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: "${{ inputs.python-version }}"

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-prefix }}

      - name: "Set `TIMESTAMP` environment variable"
        shell: bash
        run: |
          echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV"

      - name: Free Disk Space Before Build
        run: |
          echo "Disk space before cleanup:"
          df -h
          sudo rm -rf /usr/local/.ghcup
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo rm -rf /usr/local/lib/android/sdk/ndk
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/local/share/boost
          sudo apt-get clean
          echo "Disk space after cleanup:"
          df -h

      - name: Setup Salt Version
        run: |
          echo "${{ inputs.salt-version }}" > salt/_version.txt

      - name: Download Onedir Tarball as an Artifact
        uses: actions/download-artifact@v4
        with:
          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ matrix.platform }}-${{ matrix.arch }}.tar.xz
          path: artifacts/

      - name: Decompress Onedir Tarball
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ matrix.platform }}-${{ matrix.arch }}.tar.xz

      - name: "Configure docker"
        run: |
          sudo sed -i '/LimitNOFILE=infinity/c\LimitNOFILE=1048576' /lib/systemd/system/containerd.service
          sudo systemctl daemon-reload
          sudo systemctl restart containerd
          cat /lib/systemd/system/containerd.service

      - name: "Pull container ${{ matrix.container }}"
        run: |
          docker pull ${{ matrix.container }}

      - name: "Create docker network"
        run: |
           docker network create -o "com.docker.network.driver.mtu=1500" --ipv6 --subnet 2001:db8::/64 ip6net

      - name: "Host network config"
        run: |
          ip a

      - name: Free Memory Before Container
        shell: bash
        run: |
          free -h

      - name: "Create container ${{ matrix.container }}"
        env:
          GITHUB_ACTIONS: true
          CI: true
          SKIP_REQUIREMENTS_INSTALL: 1
          PRINT_TEST_SELECTION: 0
          PRINT_TEST_PLAN_ONLY: 0
          PRINT_SYSTEM_INFO: 0
          RERUN_FAILURES: 1
          GITHUB_ACTIONS_PIPELINE: 1
          SKIP_INITIAL_ONEDIR_FAILURES: 1
          SKIP_INITIAL_GH_ACTIONS_FAILURES: 1
          SKIP_CODE_COVERAGE: ${{ inputs.skip-code-coverage && '1' || '0' }}
          CONVERAGE_CONTEXT: ${{ matrix.slug }}
          COLUMNS: 190
          PIP_INDEX_URL: "${{ vars.PIP_INDEX_URL }}"
          PIP_TRUSTED_HOST: "${{ vars.PIP_TRUSTED_HOST }}"
          PIP_EXTRA_INDEX_URL: "${{ vars.PIP_EXTRA_INDEX_URL }}"
          PIP_DISABLE_PIP_VERSION_CHECK: 1
          RAISE_DEPRECATIONS_RUNTIME_ERRORS: 1
          SALT_TRANSPORT: ${{ matrix.transport }}
        run: |
          tools container create ${{ matrix.container }} --name ${{ github.run_id }}_salt-test

      - name: "Start container ${{ matrix.container }}"
        run: |
          /usr/bin/docker start ${{ github.run_id }}_salt-test

      - name: "Show container inspect ${{ matrix.container }}"
        run: |
          /usr/bin/docker inspect ${{ github.run_id }}_salt-test

      - name: Download nox.linux.${{ matrix.arch }}.tar.* artifact for session ${{ inputs.nox-session }}
        uses: actions/download-artifact@v4
        with:
          name: nox-linux-${{ matrix.arch }}-${{ inputs.nox-session }}

      - name: Install Nox
        run: |
          python3 -m pip install 'nox==${{ inputs.nox-version }}'
        env:
          PIP_INDEX_URL: https://pypi.org/simple

      - name: Decompress .nox Directory
        run: |
          docker exec ${{ github.run_id}}_salt-test python3 -m nox --force-color -e decompress-dependencies -- linux ${{ matrix.arch }}

      - name: Download testrun-changed-files.txt
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        uses: actions/download-artifact@v4
        with:
          name: testrun-changed-files.txt

      - name: Current Directory
        run: |
          pwd

      - name: Show System Info
        run: |
          docker exec -e SKIP_REQUIREMENTS_INSTALL=1 -e PRINT_SYSTEM_INFO_ONLY=1 ${{ github.run_id}}_salt-test \
          python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }}

      - name: Free Space on Host
        shell: bash
        run: |
          df -h

      - name: Show container network info
        shell: bash
        run: |
          docker exec ${{ github.run_id}}_salt-test ip addr

      - name: Show container processes
        shell: bash
        run: |
          docker exec ${{ github.run_id}}_salt-test ps auxf

      - name: Free Space on Container
        shell: bash
        run: |
          docker exec ${{ github.run_id}}_salt-test df -h

      - name: Free Memory
        shell: bash
        run: |
          free -h

      - name: Configure apparmor
        run: |
          # Apparmor's unix-chkpwd profile gets in the way of tests needing to
          # authenticate from inside a container.
          cat <<'EOF' | sudo tee /etc/apparmor.d/unix-chkpwd
          abi <abi/4.0>,
          include <tunables/global>
          profile unix-chkpwd /{,usr/}{,s}bin/unix_chkpwd flags=(unconfined) {
            include <abstractions/base>
            include <abstractions/nameservice>
            # To write records to the kernel auditing log.
            capability audit_write,
            network netlink raw,
            /{,usr/}{,s}bin/unix_chkpwd mr,
            /etc/shadow r,
            # systemd userdb, used in nspawn
            /run/host/userdb/*.user r,
            /run/host/userdb/*.user-privileged r,
            # file_inherit
            owner /dev/tty[0-9]* rw,
            include if exists <local/unix-chkpwd>
          }
          EOF
          sudo systemctl restart apparmor
          sudo aa-status

      - name: Run Changed Tests
        id: run-fast-changed-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        run: |
          docker exec ${{ github.run_id}}_salt-test python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            --core-tests --slow-tests --suppress-no-test-exit-code --from-filenames=testrun-changed-files.txt

      - name: Run Fast Tests
        id: run-fast-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }}
        run: |
          docker exec ${{ github.run_id}}_salt-test \
            python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            --suppress-no-test-exit-code

      - name: Run Slow Tests
        id: run-slow-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }}
        run: |
          docker exec ${{ github.run_id}}_salt-test \
            python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            --suppress-no-test-exit-code --no-fast-tests --slow-tests

      - name: Run Core Tests
        id: run-core-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }}
        run: |
          docker exec ${{ github.run_id}}_salt-test \
            python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            --suppress-no-test-exit-code --no-fast-tests --core-tests

      - name: Run Flaky Tests
        id: run-flaky-tests
        if: ${{ fromJSON(inputs.testrun)['selected_tests']['flaky'] }}
        run: |
          docker exec ${{ github.run_id}}_salt-test \
            python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            --suppress-no-test-exit-code --no-fast-tests --flaky-jail

      - name: Run Full Tests
        id: run-full-tests
        if: ${{ fromJSON(inputs.testrun)['type'] == 'full' }}
        run: |
          docker exec ${{ github.run_id}}_salt-test \
            python3 -m nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            --slow-tests --core-tests --test-group-count=${{ matrix.test-group-count || 1 }} --test-group=${{ matrix.test-group || 1 }}

      - name: Stop Container
        run: |
          docker container stop ${{ github.run_id}}_salt-test

      - name: Remove Container
        run: |
          docker container rm ${{ github.run_id}}_salt-test

      - name: Remove Container Image
        run: |
          docker image rm ${{ matrix.container }}

      - name: Fix file ownership
        run: |
          sudo chown -R "$(id -un)" .

      - name: Combine Coverage Reports
        if: always() && inputs.skip-code-coverage == false
        run: |
          nox --force-color -e combine-coverage

      - name: Prepare Test Run Artifacts
        id: download-artifacts-from-vm
        if: always()
        run: |
          # Delete the salt onedir, we won't need it anymore and it will prevent
          # from it showing in the tree command below
          sudo rm -rf artifacts/salt*
          tree -a artifacts
          if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then
            mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ matrix.slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}
          fi

      - name: Upload Code Coverage Test Run Artifacts
        if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-coverage-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/coverage/
          include-hidden-files: true

      - name: Upload JUnit XML Test Run Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-junit-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}${{ matrix.fips && '(fips)' || '' }}-${{ matrix.tests-chunk }}-${{ matrix.test-group || 1 }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/xml-unittests-output/
          include-hidden-files: true

      - name: Upload Test Run Log Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-log-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}${{ matrix.fips && '(fips)' || '' }}-${{ matrix.tests-chunk }}-${{ matrix.test-group || 1 }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/logs
          include-hidden-files: true

  test-macos:
    name: ${{ matrix.display_name }} ${{ matrix.tests-chunk }} ${{ matrix.transport }}${{ matrix.test-group && ' ' || '' }}${{ matrix.test-group && matrix.test-group || '' }}
    runs-on: ${{ matrix.runner }}
    # Full test runs. Each chunk should never take more than 2 hours.
    # Partial test runs(no chunk parallelization), 6 Hours
    if: ${{ !cancelled() && toJSON(fromJSON(inputs.matrix)['macos']) != '[]' }}
    timeout-minutes: ${{ fromJSON(inputs.testrun)['type'] == 'full' && inputs.default-timeout || 360 }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(inputs.matrix)['macos'] }}
    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t"

      - name: "Set `TIMESTAMP` environment variable"
        shell: bash
        run: |
          echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Setup Python ${{ inputs.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: "${{ inputs.python-version }}"

      - name: Setup Salt Version
        run: |
          echo "${{ inputs.salt-version }}" > salt/_version.txt

      - name: Download Onedir Tarball as an Artifact
        uses: actions/download-artifact@v4
        with:
          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ matrix.platform }}-${{ matrix.arch }}.tar.xz
          path: artifacts/

      - name: Decompress Onedir Tarball
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ matrix.platform }}-${{ matrix.arch }}.tar.xz

      - name: Install System Dependencies
        run: |
          brew install tree

      - name: Download nox.macos.${{ matrix.arch }}.tar.* artifact for session ${{ inputs.nox-session }}
        uses: actions/download-artifact@v4
        with:
          name: nox-macos-${{ matrix.arch }}-${{ inputs.nox-session }}
      - name: Install Nox
        run: |
          python3 -m pip install 'nox==${{ inputs.nox-version }}'
        env:
          PIP_INDEX_URL: https://pypi.org/simple

      - name: Decompress .nox Directory
        run: |
          nox --force-color -e decompress-dependencies -- macos ${{ matrix.arch }}

      - name: Download testrun-changed-files.txt
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        uses: actions/download-artifact@v4
        with:
          name: testrun-changed-files.txt

      - name: Show System Info
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_SYSTEM_INFO_ONLY: "1"
        run: |
          sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }}

      - name: Run Changed Tests
        id: run-fast-changed-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_TEST_SELECTION: "0"
          PRINT_TEST_PLAN_ONLY: "0"
          PRINT_SYSTEM_INFO: "0"
          RERUN_FAILURES: "1"
          GITHUB_ACTIONS_PIPELINE: "1"
          SKIP_INITIAL_ONEDIR_FAILURES: "1"
          SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
          SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
          COVERAGE_CONTEXT: ${{ matrix.slug }}
          COLUMNS: 190
          PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
          PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
          PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
          PIP_DISABLE_PIP_VERSION_CHECK: "1"
          RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
          SALT_TRANSPORT: ${{ matrix.transport }}
        run: |
          sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            -k "mac or darwin" --core-tests --slow-tests --suppress-no-test-exit-code \
            --from-filenames=testrun-changed-files.txt

      - name: Run Fast Tests
        id: run-fast-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }}
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_TEST_SELECTION: "0"
          PRINT_TEST_PLAN_ONLY: "0"
          PRINT_SYSTEM_INFO: "0"
          RERUN_FAILURES: "1"
          GITHUB_ACTIONS_PIPELINE: "1"
          SKIP_INITIAL_ONEDIR_FAILURES: "1"
          SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
          SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
          COVERAGE_CONTEXT: ${{ matrix.slug }}
          COLUMNS: 190
          PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
          PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
          PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
          PIP_DISABLE_PIP_VERSION_CHECK: "1"
          RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
          SALT_TRANSPORT: ${{ matrix.transport }}
        run: |
          sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            -k "mac or darwin" --suppress-no-test-exit-code

      - name: Run Slow Tests
        id: run-slow-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }}
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_TEST_SELECTION: "0"
          PRINT_TEST_PLAN_ONLY: "0"
          PRINT_SYSTEM_INFO: "0"
          RERUN_FAILURES: "1"
          GITHUB_ACTIONS_PIPELINE: "1"
          SKIP_INITIAL_ONEDIR_FAILURES: "1"
          SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
          SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
          COVERAGE_CONTEXT: ${{ matrix.slug }}
          COLUMNS: 190
          PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
          PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
          PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
          PIP_DISABLE_PIP_VERSION_CHECK: "1"
          RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
          SALT_TRANSPORT: ${{ matrix.transport }}
        run: |
          sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            -k "mac or darwin" --suppress-no-test-exit-code --no-fast-tests --slow-tests

      - name: Run Core Tests
        id: run-core-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }}
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_TEST_SELECTION: "0"
          PRINT_TEST_PLAN_ONLY: "0"
          PRINT_SYSTEM_INFO: "0"
          RERUN_FAILURES: "1"
          GITHUB_ACTIONS_PIPELINE: "1"
          SKIP_INITIAL_ONEDIR_FAILURES: "1"
          SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
          SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
          COVERAGE_CONTEXT: ${{ matrix.slug }}
          COLUMNS: 190
          PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
          PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
          PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
          PIP_DISABLE_PIP_VERSION_CHECK: "1"
          RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
          SALT_TRANSPORT: ${{ matrix.transport }}
        run: |
          sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            -k "mac or darwin" --suppress-no-test-exit-code --no-fast-tests --core-tests

      - name: Run Flaky Tests
        id: run-flaky-tests
        if: ${{ fromJSON(inputs.testrun)['selected_tests']['flaky'] }}
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_TEST_SELECTION: "0"
          PRINT_TEST_PLAN_ONLY: "0"
          PRINT_SYSTEM_INFO: "0"
          RERUN_FAILURES: "1"
          GITHUB_ACTIONS_PIPELINE: "1"
          SKIP_INITIAL_ONEDIR_FAILURES: "1"
          SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
          SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
          COVERAGE_CONTEXT: ${{ matrix.slug }}
          COLUMNS: 190
          PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
          PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
          PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
          PIP_DISABLE_PIP_VERSION_CHECK: "1"
          RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
          SALT_TRANSPORT: ${{ matrix.transport }}
        run: |
          sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            -k "mac or darwin" --suppress-no-test-exit-code --no-fast-tests --flaky-jail

      - name: Run Full Tests
        id: run-full-tests
        if: ${{ fromJSON(inputs.testrun)['type'] == 'full' }}
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_TEST_SELECTION: "0"
          PRINT_TEST_PLAN_ONLY: "0"
          PRINT_SYSTEM_INFO: "0"
          RERUN_FAILURES: "1"
          GITHUB_ACTIONS_PIPELINE: "1"
          SKIP_INITIAL_ONEDIR_FAILURES: "1"
          SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
          SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
          COVERAGE_CONTEXT: ${{ matrix.slug }}
          COLUMNS: 190
          PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
          PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
          PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
          PIP_DISABLE_PIP_VERSION_CHECK: "1"
          RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
          SALT_TRANSPORT: ${{ matrix.transport }}
        run: |
          sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \
            --slow-tests --core-tests -k "mac or darwin" --test-group-count=${{ matrix.test-group-count || 1 }} --test-group=${{ matrix.test-group || 1 }}


      - name: Fix file ownership
        run: |
          sudo chown -R "$(id -un)" .

      - name: Combine Coverage Reports
        if: always() && inputs.skip-code-coverage == false
        run: |
          nox --force-color -e combine-coverage

      - name: Prepare Test Run Artifacts
        id: download-artifacts-from-vm
        if: always()
        run: |
          # Delete the salt onedir, we won't need it anymore and it will prevent
          # from it showing in the tree command below
          rm -rf artifacts/salt*
          tree -a artifacts
          if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then
            mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ matrix.slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}
          fi

      - name: Upload Code Coverage Test Run Artifacts
        if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-coverage-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ matrix.test-group || 1 }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/coverage/
          include-hidden-files: true

      - name: Upload JUnit XML Test Run Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-junit-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ matrix.test-group || 1 }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/xml-unittests-output/
          include-hidden-files: true

      - name: Upload Test Run Log Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-log-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ matrix.test-group || 1 }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/logs
          include-hidden-files: true

  test-windows:
    name: ${{ matrix.display_name }} ${{ matrix.tests-chunk }} ${{ matrix.transport }}${{ matrix.test-group && ' ' || '' }}${{ matrix.test-group && matrix.test-group || '' }}

    if: ${{ !cancelled() && toJSON(fromJSON(inputs.matrix)['windows']) != '[]' }}
    runs-on: ${{ matrix.slug }}
    # Full test runs. Each chunk should never take more than 2 hours.
    # Partial test runs(no chunk parallelization), 6 Hours
    timeout-minutes: ${{ fromJSON(inputs.testrun)['type'] == 'full' && inputs.default-timeout || 360 }}
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(inputs.matrix)['windows'] }}
    steps:
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t"

      - name: "Set `TIMESTAMP` environment variable"
        shell: bash
        run: |
          echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Setup Salt Version
        run: |
          echo "${{ inputs.salt-version }}" > salt/_version.txt

      - name: Download Onedir Tarball as an Artifact
        uses: actions/download-artifact@v4
        with:
          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ matrix.platform }}-${{ matrix.arch }}.tar.xz
          path: artifacts/

      - name: Decompress Onedir Tarball
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ matrix.platform }}-${{ matrix.arch }}.tar.xz

      - name: Install System Dependencies
        run: |
          echo true

      - name: Setup Python ${{ inputs.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: "${{ inputs.python-version }}"

      - name: Install Nox
        run: |
          python3 -m pip install 'nox==${{ inputs.nox-version }}'
        env:
          PIP_INDEX_URL: https://pypi.org/simple

      - name: Download nox.windows.${{ matrix.arch }}.tar.* artifact for session ${{ inputs.nox-session }}
        uses: actions/download-artifact@v4
        with:
          name: nox-windows-${{ matrix.arch }}-${{ inputs.nox-session }}

      - name: Decompress .nox Directory
        run: |
          nox --force-color -e decompress-dependencies -- windows ${{ matrix.arch }}

      - name: Download testrun-changed-files.txt
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        uses: actions/download-artifact@v4
        with:
          name: testrun-changed-files.txt

      - name: Check nox python
        continue-on-error: true
        run: |
          .nox/ci-test-onedir/Scripts/python.exe --version

      - name: Show System Info
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_SYSTEM_INFO_ONLY: "1"
        run: |
          nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }}

      - name: Run Changed Tests
        id: run-fast-changed-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_TEST_SELECTION: "0"
          PRINT_TEST_PLAN_ONLY: "0"
          PRINT_SYSTEM_INFO: "0"
          RERUN_FAILURES: "1"
          GITHUB_ACTIONS_PIPELINE: "1"
          SKIP_INITIAL_ONEDIR_FAILURES: "1"
          SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
          SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
          COVERAGE_CONTEXT: ${{ matrix.slug }}
          COLUMNS: 190
          PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
          PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
          PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
          PIP_DISABLE_PIP_VERSION_CHECK: "1"
          RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
          SALT_TRANSPORT: ${{ matrix.transport }}
          TMPDIR: ${{ runner.temp }}
        shell: powershell
        run: >
          nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} --
          -k "win" --core-tests --slow-tests --suppress-no-test-exit-code
          "--from-filenames=testrun-changed-files.txt"

      - name: Run Fast Tests
        id: run-fast-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }}
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_TEST_SELECTION: "0"
          PRINT_TEST_PLAN_ONLY: "0"
          PRINT_SYSTEM_INFO: "0"
          RERUN_FAILURES: "1"
          GITHUB_ACTIONS_PIPELINE: "1"
          SKIP_INITIAL_ONEDIR_FAILURES: "1"
          SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
          SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
          COVERAGE_CONTEXT: ${{ matrix.slug }}
          COLUMNS: 190
          PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
          PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
          PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
          PIP_DISABLE_PIP_VERSION_CHECK: "1"
          RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
          SALT_TRANSPORT: ${{ matrix.transport }}
          TMPDIR: ${{ runner.temp }}
        shell: powershell
        run: >
          nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} --
          -k "win" --suppress-no-test-exit-code

      - name: Run Slow Tests
        id: run-slow-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }}
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_TEST_SELECTION: "0"
          PRINT_TEST_PLAN_ONLY: "0"
          PRINT_SYSTEM_INFO: "0"
          RERUN_FAILURES: "1"
          GITHUB_ACTIONS_PIPELINE: "1"
          SKIP_INITIAL_ONEDIR_FAILURES: "1"
          SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
          SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
          COVERAGE_CONTEXT: ${{ matrix.slug }}
          COLUMNS: 190
          PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
          PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
          PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
          PIP_DISABLE_PIP_VERSION_CHECK: "1"
          RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
          SALT_TRANSPORT: ${{ matrix.transport }}
          TMPDIR: ${{ runner.temp }}
        shell: powershell
        run: >
          nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} --
          -k "win" --suppress-no-test-exit-code --no-fast-tests --slow-tests

      - name: Run Core Tests
        id: run-core-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }}
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_TEST_SELECTION: "0"
          PRINT_TEST_PLAN_ONLY: "0"
          PRINT_SYSTEM_INFO: "0"
          RERUN_FAILURES: "1"
          GITHUB_ACTIONS_PIPELINE: "1"
          SKIP_INITIAL_ONEDIR_FAILURES: "1"
          SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
          SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
          COVERAGE_CONTEXT: ${{ matrix.slug }}
          COLUMNS: 190
          PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
          PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
          PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
          PIP_DISABLE_PIP_VERSION_CHECK: "1"
          RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
          SALT_TRANSPORT: ${{ matrix.transport }}
          TMPDIR: ${{ runner.temp }}
        shell: powershell
        run: >
          nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} --
          -k "win" --suppress-no-test-exit-code --no-fast-tests --core-tests

      - name: Run Flaky Tests
        id: run-flaky-tests
        if: ${{ fromJSON(inputs.testrun)['selected_tests']['flaky'] }}
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_TEST_SELECTION: "0"
          PRINT_TEST_PLAN_ONLY: "0"
          PRINT_SYSTEM_INFO: "0"
          RERUN_FAILURES: "1"
          GITHUB_ACTIONS_PIPELINE: "1"
          SKIP_INITIAL_ONEDIR_FAILURES: "1"
          SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
          SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
          COVERAGE_CONTEXT: ${{ matrix.slug }}
          COLUMNS: 190
          PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
          PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
          PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
          PIP_DISABLE_PIP_VERSION_CHECK: "1"
          RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
          SALT_TRANSPORT: ${{ matrix.transport }}
          TMPDIR: ${{ runner.temp }}
        shell: powershell
        run: >
          nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} --
          -k "win" --suppress-no-test-exit-code --no-fast-tests --flaky-jail

      - name: Run Full Tests
        id: run-full-tests
        if: ${{ fromJSON(inputs.testrun)['type'] == 'full' }}
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_TEST_SELECTION: "0"
          PRINT_TEST_PLAN_ONLY: "0"
          PRINT_SYSTEM_INFO: "0"
          RERUN_FAILURES: "1"
          GITHUB_ACTIONS_PIPELINE: "1"
          SKIP_INITIAL_ONEDIR_FAILURES: "1"
          SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
          SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
          COVERAGE_CONTEXT: ${{ matrix.slug }}
          COLUMNS: 190
          PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
          PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
          PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
          PIP_DISABLE_PIP_VERSION_CHECK: "1"
          RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
          SALT_TRANSPORT: ${{ matrix.transport }}
          TMPDIR: ${{ runner.temp }}
        shell: powershell
        run: >
          nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} --
          --slow-tests --core-tests -k "win" --test-group-count=${{ matrix.test-group-count || 1 }} --test-group=${{ matrix.test-group || 1 }}


      - name: Combine Coverage Reports
        if: always() && inputs.skip-code-coverage == false
        run: |
          nox --force-color -e combine-coverage

      - name: Prepare Test Run Artifacts
        id: download-artifacts-from-vm
        if: always()
        shell: bash
        run: |
          # Delete the salt onedir, we won't need it anymore and it will prevent
          # from it showing in the tree command below
          rm -rf artifacts/salt*
          if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then
            mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ matrix.slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}
          fi

      - name: Upload Code Coverage Test Run Artifacts
        if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-coverage-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ matrix.test-group || 1 }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/coverage/
          include-hidden-files: true

      - name: Upload JUnit XML Test Run Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-junit-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ matrix.test-group || 1 }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/xml-unittests-output/
          include-hidden-files: true

      - name: Upload Test Run Log Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-log-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ matrix.test-group || 1 }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/logs
          include-hidden-files: true


  report:
    name: Test Reports
    runs-on: ubuntu-22.04
    if: ${{ false }}
    needs:
      - test-linux
      - test-linux-arm64
      - test-macos
      - test-windows
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(inputs.matrix)['linux'] }}
    env:
      PIP_INDEX_URL: https://pypi.org/simple

    steps:
      - name: Checkout Source Code
        uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"

      - name: Merge JUnit XML Test Run Artifacts
        continue-on-error: true
        uses: actions/upload-artifact/merge@v4
        with:
          name: testrun-junit-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}
          pattern: testrun-junit-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-*
          separate-directories: false
          delete-merged: true

      - name: Merge Log Test Run Artifacts
        continue-on-error: true
        uses: actions/upload-artifact/merge@v4
        with:
          name: testrun-log-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}
          pattern: testrun-log-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-*
          separate-directories: false
          delete-merged: true

      - name: Merge Code Coverage Test Run Artifacts
        if: ${{ inputs.skip-code-coverage == false }}
        continue-on-error: true
        uses: actions/upload-artifact/merge@v4
        with:
          name: testrun-coverage-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}
          pattern: testrun-coverage-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-*
          separate-directories: false
          delete-merged: true

      - name: Download Code Coverage Test Run Artifacts
        uses: actions/download-artifact@v4
        if: ${{ inputs.skip-code-coverage == false }}
        id: download-coverage-artifacts
        with:
          path: artifacts/coverage/
          pattern: testrun-coverage-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}*
          merge-multiple: true

      - name: Show Downloaded Test Run Artifacts
        if: ${{ inputs.skip-code-coverage == false }}
        run: |
          tree -a artifacts

      - name: Setup Python ${{ inputs.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: "${{ inputs.python-version }}"

      - name: Install Nox
        run: |
          python3 -m pip install 'nox==${{ inputs.nox-version }}'

      - name: Create XML Coverage Reports
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success' && job.status != 'cancelled'
        run: |
          nox --force-color -e create-xml-coverage-reports
          mv artifacts/coverage/salt.xml artifacts/coverage/salt..${{ matrix.slug }}..${{ inputs.nox-session }}.xml
          mv artifacts/coverage/tests.xml artifacts/coverage/tests..${{ matrix.slug }}..${{ inputs.nox-session }}.xml

      - name: Report Salt Code Coverage
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        continue-on-error: true
        run: |
          nox --force-color -e report-coverage -- salt

      - name: Report Combined Code Coverage
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        continue-on-error: true
        run: |
          nox --force-color -e report-coverage

      - name: Rename Code Coverage DB
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        continue-on-error: true
        run: |
          mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ matrix.slug }}.${{ inputs.nox-session }}

      - name: Upload Code Coverage DB
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: all-testrun-coverage-artifacts-${{ matrix.slug }}.${{ inputs.nox-session }}
          path: artifacts/coverage
          include-hidden-files: true