---
name: Test Artifact

on:
  workflow_call:
    inputs:
      distro-slug:
        required: true
        type: string
        description: The OS slug to run tests against
      nox-session:
        required: true
        type: string
        description: The nox session to run
      testrun:
        required: true
        type: string
        description: JSON string containing information about what and how to run the test suite
      salt-version:
        type: string
        required: true
        description: The Salt version to set prior to running tests.
      cache-prefix:
        required: true
        type: string
        description: Seed used to invalidate caches
      platform:
        required: true
        type: string
        description: The platform being tested
      arch:
        required: true
        type: string
        description: The platform arch being tested
      nox-version:
        required: true
        type: string
        description: The nox version to install
      timeout-minutes:
        required: true
        type: number
        description: Timeout, in minutes, for the test job
      gh-actions-python-version:
        required: false
        type: string
        description: The python version to run tests with
        default: "3.10"
      fips:
        required: false
        type: boolean
        default: false
        description: Test run with FIPS enabled
      package-name:
        required: false
        type: string
        description: The onedir package name to use
        default: salt
      skip-code-coverage:
        required: false
        type: boolean
        description: Skip code coverage
        default: false
      workflow-slug:
        required: false
        type: string
        description: Which workflow is running.
        default: ci

env:
  COLUMNS: 190
  AWS_MAX_ATTEMPTS: "10"
  AWS_RETRY_MODE: "adaptive"
  PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
  PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
  PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
  PIP_DISABLE_PIP_VERSION_CHECK: "1"
  RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"

jobs:

  generate-matrix:
    name: Test Matrix
    runs-on: ubuntu-latest
    outputs:
      matrix-include: ${{ steps.generate-matrix.outputs.matrix }}
      build-reports: ${{ steps.generate-matrix.outputs.build-reports }}
    steps:
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-prefix }}
        env:
          PIP_INDEX_URL: https://pypi.org/simple

      - name: Generate Test Matrix
        id: generate-matrix
        run: |
          tools ci matrix --workflow=${{ inputs.workflow-slug }} ${{ fromJSON(inputs.testrun)['type'] == 'full' && '--full ' || '' }}${{ inputs.fips && '--fips ' || '' }}${{ inputs.distro-slug }}

  test:
    name: Test
    runs-on:
      - self-hosted
      - linux
      - bastion
    timeout-minutes: ${{ inputs.timeout-minutes }}
    needs:
      - generate-matrix
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include) }}
    env:
      SALT_TRANSPORT: ${{ matrix.transport }}
      TEST_GROUP: ${{ matrix.test-group || 1 }}

    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t"

      - name: "Set `TIMESTAMP` environment variable"
        shell: bash
        run: |
          echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Setup Salt Version
        run: |
          echo "${{ inputs.salt-version }}" > salt/_version.txt

      - name: Download Onedir Tarball as an Artifact
        uses: actions/download-artifact@v4
        with:
          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
          path: artifacts/

      - name: Decompress Onedir Tarball
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz

      - name: Download nox.windows.${{ inputs.arch }}.tar.* artifact for session ${{ inputs.nox-session }}
        uses: actions/download-artifact@v4
        with:
          name: nox-windows-${{ inputs.arch }}-${{ inputs.nox-session }}

      - name: PyPi Proxy
        run: |
          sed -i '7s;^;--index-url=${{ vars.PIP_INDEX_URL }} --trusted-host ${{ vars.PIP_TRUSTED_HOST }} --extra-index-url=${{ vars.PIP_EXTRA_INDEX_URL }}\n;' requirements/static/ci/*/*.txt

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-prefix }}

      - name: Download testrun-changed-files.txt
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        uses: actions/download-artifact@v4
        with:
          name: testrun-changed-files.txt

      - name: Get Salt Project GitHub Actions Bot Environment
        run: |
          TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
          SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
          echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"

      - name: Start VM
        id: spin-up-vm
        env:
          TESTS_CHUNK: ${{ matrix.tests-chunk }}
        run: |
          tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ inputs.distro-slug }}

      - name: List Free Space
        run: |
          tools --timestamps vm ssh ${{ inputs.distro-slug }} -- df -h || true

      - name: Upload Checkout To VM
        run: |
          tools --timestamps vm rsync ${{ inputs.distro-slug }}

      - name: Decompress .nox Directory
        run: |
          tools --timestamps vm decompress-dependencies ${{ inputs.distro-slug }}

      - name: Show System Info
        run: |
          tools --timestamps --timeout-secs=1800 vm test --skip-requirements-install --print-system-information-only \
            --nox-session=${{ inputs.nox-session }} ${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }}

      - name: Run Changed Tests
        id: run-fast-changed-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ matrix.fips && '--fips ' || '' }}${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }} -- --core-tests --slow-tests --suppress-no-test-exit-code \
            --from-filenames=testrun-changed-files.txt

      - name: Run Fast Tests
        id: run-fast-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ (inputs.skip-code-coverage && matrix.tests-chunk != 'unit') && '--skip-code-coverage' || '' }} \
            ${{ matrix.fips && '--fips ' || '' }}${{ inputs.distro-slug }} ${{ matrix.tests-chunk }}

      - name: Run Slow Tests
        id: run-slow-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ matrix.fips && '--fips ' || '' }}${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }} -- --no-fast-tests --slow-tests

      - name: Run Core Tests
        id: run-core-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ matrix.fips && '--fips ' || '' }}${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }} -- --no-fast-tests --core-tests

      - name: Run Flaky Tests
        id: run-flaky-tests
        if: ${{ fromJSON(inputs.testrun)['selected_tests']['flaky'] }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ matrix.fips && '--fips ' || '' }}${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }} -- --no-fast-tests --flaky-jail

      - name: Run Full Tests
        id: run-full-tests
        if: ${{ fromJSON(inputs.testrun)['type'] == 'full' }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ (inputs.skip-code-coverage && matrix.tests-chunk != 'unit') && '--skip-code-coverage' || '' }} \
            -E TEST_GROUP ${{ matrix.fips && '--fips ' || '' }}${{ inputs.distro-slug }} ${{ matrix.tests-chunk }} -- --slow-tests --core-tests \
            --test-group-count=${{ matrix.test-group-count || 1 }} --test-group=${{ matrix.test-group || 1 }}

      - name: Combine Coverage Reports
        if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success'
        run: |
          tools --timestamps vm combine-coverage ${{ inputs.distro-slug }}

      - name: Download Test Run Artifacts
        id: download-artifacts-from-vm
        if: always() && steps.spin-up-vm.outcome == 'success'
        run: |
          tools --timestamps vm download-artifacts ${{ inputs.distro-slug }}
          # Delete the salt onedir, we won't need it anymore and it will prevent
          # from it showing in the tree command below
          rm -rf artifacts/salt*
          tree -a artifacts
          if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then
            mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}.grp${{ matrix.test-group || '1' }}
          fi

      - name: Destroy VM
        if: always()
        run: |
          tools --timestamps vm destroy --no-wait ${{ inputs.distro-slug }} || true

      - name: Upload Code Coverage Test Run Artifacts
        if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/coverage/
          include-hidden-files: true

      - name: Upload JUnit XML Test Run Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/xml-unittests-output/
          include-hidden-files: true

      - name: Upload Test Run Log Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/logs
          include-hidden-files: true


  report:
    name: Test Reports
    if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && needs.test.result != 'cancelled' && needs.test.result != 'skipped'
    runs-on: ubuntu-latest
    needs:
      - test
      - generate-matrix
    env:
      PIP_INDEX_URL: https://pypi.org/simple

    steps:
      - name: Checkout Source Code
        uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"

      - name: Merge JUnit XML Test Run Artifacts
        if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped'
        continue-on-error: true
        uses: actions/upload-artifact/merge@v4
        with:
          name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}
          pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-*
          separate-directories: false
          delete-merged: true

      - name: Merge Log Test Run Artifacts
        if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped'
        continue-on-error: true
        uses: actions/upload-artifact/merge@v4
        with:
          name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}
          pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-*
          separate-directories: false
          delete-merged: true

      - name: Merge Code Coverage Test Run Artifacts
        if: ${{ inputs.skip-code-coverage == false }}
        continue-on-error: true
        uses: actions/upload-artifact/merge@v4
        with:
          name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}
          pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-*
          separate-directories: false
          delete-merged: true

      - name: Download Code Coverage Test Run Artifacts
        uses: actions/download-artifact@v4
        if: ${{ inputs.skip-code-coverage == false }}
        id: download-coverage-artifacts
        with:
          path: artifacts/coverage/
          pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}*
          merge-multiple: true

      - name: Show Downloaded Test Run Artifacts
        if: ${{ inputs.skip-code-coverage == false }}
        run: |
          tree -a artifacts

      - name: Install Nox
        run: |
          python3 -m pip install 'nox==${{ inputs.nox-version }}'

      - name: Create XML Coverage Reports
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success' && job.status != 'cancelled'
        run: |
          nox --force-color -e create-xml-coverage-reports
          mv artifacts/coverage/salt.xml artifacts/coverage/salt..${{ inputs.distro-slug }}..${{ inputs.nox-session }}.xml || true
          mv artifacts/coverage/tests.xml artifacts/coverage/tests..${{ inputs.distro-slug }}..${{ inputs.nox-session }}.xml || true

      - name: Report Salt Code Coverage
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        continue-on-error: true
        run: |
          nox --force-color -e report-coverage -- salt

      - name: Report Combined Code Coverage
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        continue-on-error: true
        run: |
          nox --force-color -e report-coverage

      - name: Rename Code Coverage DB
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        continue-on-error: true
        run: |
          mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}

      - name: Upload Code Coverage DB
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: all-testrun-coverage-artifacts-${{ inputs.distro-slug }}.${{ inputs.nox-session }}
          path: artifacts/coverage
          include-hidden-files: true