salt/.github/workflows/test-action.yml

name: Test Artifact

on:
  workflow_call:
    inputs:
      distro-slug:
        required: true
        type: string
        description: The OS slug to run tests against
      nox-session:
        required: true
        type: string
        description: The nox session to run
      testrun:
        required: true
        type: string
        description: JSON string containing information about what and how to run the test suite
      salt-version:
        type: string
        required: true
        description: The Salt version to set prior to running tests.
      cache-seed:
        required: true
        type: string
        description: Seed used to invalidate caches
      platform:
        required: true
        type: string
        description: The platform being tested
      arch:
        required: true
        type: string
        description: The platform arch being tested
      package-name:
        required: false
        type: string
        description: The onedir package name to use
        default: salt


env:
  NOX_VERSION: "2022.8.7"
  COLUMNS: 160
  AWS_MAX_ATTEMPTS: "10"
  AWS_RETRY_MODE: "adaptive"

jobs:

  generate-matrix:
    name: Generate Test Matrix
    runs-on:
      - self-hosted
      - linux
      - x86_64
    outputs:
      matrix-include: ${{ steps.generate-matrix.outputs.matrix }}
      transport-matrix-include: ${{ steps.generate-transport-matrix.outputs.matrix }}
    steps:
      - name: Checkout Source Code
        uses: actions/checkout@v3

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts

      - name: Generate Test Matrix
        id: generate-matrix
        run: |
          TEST_MATRIX=$(tools ci matrix ${{ inputs.distro-slug }})
          echo "$TEST_MATRIX"
          echo "matrix=$TEST_MATRIX" >> "$GITHUB_OUTPUT"

      - name: Generate Transport Matrix
        id: generate-transport-matrix
        run: |
          TRANSPORT_MATRIX=$(tools ci transport-matrix ${{ inputs.distro-slug }})
          echo "$TRANSPORT_MATRIX"
          echo "matrix=$TRANSPORT_MATRIX" >> "$GITHUB_OUTPUT"

  dependencies:
    name: Setup Test Dependencies
    needs:
      - generate-matrix
    runs-on:
      - self-hosted
      - linux
      - bastion
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.generate-matrix.outputs.transport-matrix-include) }}
    steps:
      - name: Checkout Source Code
        uses: actions/checkout@v3

      - name: Cache nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
        id: nox-dependencies-cache
        uses: actions/cache@v3
        with:
          path: nox.${{ inputs.distro-slug }}.tar.*
          key: ${{ inputs.cache-seed }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ matrix.transport }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }}

      - name: Download Onedir Tarball as an Artifact
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        uses: actions/download-artifact@v3
        with:
          name: ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz
          path: artifacts/

      - name: Decompress Onedir Tarball
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz

      # Skip jobs if nox.*.tar.* is already cached
      - name: PyPi Proxy
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          sed -i '7s;^;--index-url=https://pypi-proxy.saltstack.net/root/local/+simple/ --extra-index-url=https://pypi.org/simple\n;' requirements/static/ci/*/*.txt

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts

      - name: Define Nox Session
        run: |
          if [ "${{ matrix.transport }}" != "tcp" ]; then
            echo NOX_SESSION=${{ inputs.nox-session }} >> $GITHUB_ENV
          else
            echo NOX_SESSION=${{ inputs.nox-session }}-tcp >> $GITHUB_ENV
          fi

      - name: Start VM
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        id: spin-up-vm
        run: |
          tools --timestamps vm create --retries=2 ${{ inputs.distro-slug }}

      - name: List Free Space
        run: |
          tools --timestamps vm ssh ${{ inputs.distro-slug }} -- df -h || true

      - name: Upload Checkout To VM
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm rsync ${{ inputs.distro-slug }}

      - name: Install Dependencies
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm install-dependencies --nox-session=${{ env.NOX_SESSION }} ${{ inputs.distro-slug }}

      - name: Cleanup .nox Directory
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm pre-archive-cleanup ${{ inputs.distro-slug }}

      - name: Compress .nox Directory
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm compress-dependencies ${{ inputs.distro-slug }}

      - name: Download Compressed .nox Directory
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm download-dependencies ${{ inputs.distro-slug }}

      - name: Destroy VM
        if: always() && steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm destroy ${{ inputs.distro-slug }}

      - name: Set Exit Status
        if: always()
        run: |
          python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
          echo "${{ job.status }}" > exitstatus/${{ github.job }}-${{ inputs.distro-slug }}-deps

      - name: Upload Exit Status
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: exitstatus
          path: exitstatus
          if-no-files-found: error

  test:
    name: Test
    runs-on:
      - self-hosted
      - linux
      - bastion
    timeout-minutes: 240  # 4 Hours - More than this and something is wrong
    needs:
      - dependencies
      - generate-matrix
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include) }}

    steps:
      - name: Checkout Source Code
        uses: actions/checkout@v3

      - name: Setup Salt Version
        run: |
          echo "${{ inputs.salt-version }}" > salt/_version.txt

      - name: Download Onedir Tarball as an Artifact
        uses: actions/download-artifact@v3
        with:
          name: ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz
          path: artifacts/

      - name: Decompress Onedir Tarball
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz

      - name: Download cached nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
        uses: actions/cache@v3
        with:
          path: nox.${{ inputs.distro-slug }}.tar.*
          key: ${{ inputs.cache-seed }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ matrix.transport }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }}

      - name: PyPi Proxy
        run: |
          sed -i '7s;^;--index-url=https://pypi-proxy.saltstack.net/root/local/+simple/ --extra-index-url=https://pypi.org/simple\n;' requirements/static/ci/*/*.txt

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts

      - name: Define Nox Session
        run: |
          if [ "${{ matrix.transport }}" != "tcp" ]; then
            echo NOX_SESSION=${{ inputs.nox-session }} >> $GITHUB_ENV
          else
            echo NOX_SESSION=${{ inputs.nox-session }}-tcp >> $GITHUB_ENV
          fi

      - name: Download testrun-changed-files.txt
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        uses: actions/download-artifact@v3
        with:
          name: testrun-changed-files.txt

      - name: Start VM
        id: spin-up-vm
        env:
          TESTS_CHUNK: ${{ matrix.tests-chunk }}
        run: |
          tools --timestamps vm create --retries=2 ${{ inputs.distro-slug }}

      - name: List Free Space
        run: |
          tools --timestamps vm ssh ${{ inputs.distro-slug }} -- df -h || true

      - name: Upload Checkout To VM
        run: |
          tools --timestamps vm rsync ${{ inputs.distro-slug }}

      - name: Decompress .nox Directory
        run: |
          tools --timestamps vm decompress-dependencies ${{ inputs.distro-slug }}

      - name: Show System Info & Test Plan
        run: |
          tools --timestamps --no-output-timeout-secs=18000 vm testplan --skip-requirements-install \
            --nox-session=${{ env.NOX_SESSION }} ${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }}

      - name: Run Slow/Changed Tests
        id: run-slow-changed-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        run: |
          tools --timestamps --no-output-timeout-secs=18000 vm test --skip-requirements-install \
            --nox-session=${{ env.NOX_SESSION }} --rerun-failures ${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }} -- --run-slow --suppress-no-test-exit-code \
            --from-filenames=testrun-changed-files.txt

      - name: Run Fast Tests
        id: run-fast-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        run: |
          tools --timestamps --no-output-timeout-secs=18000 vm test --skip-requirements-install \
            --nox-session=${{ env.NOX_SESSION }} --rerun-failures ${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }}

      - name: Run Full Tests
        id: run-full-tests
        if: ${{ fromJSON(inputs.testrun)['type'] == 'full' }}
        run: |
          tools --timestamps --no-output-timeout-secs=18000 vm test --skip-requirements-install \
            --nox-session=${{ env.NOX_SESSION }} --rerun-failures ${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }} -- --run-slow

      - name: Combine Coverage Reports
        if: always() && steps.spin-up-vm.outcome == 'success' && contains(fromJSON('["canceled", "skipped"]'), steps.run-tests.outcome) == false
        run: |
          tools --timestamps vm combine-coverage ${{ inputs.distro-slug }}

      - name: Download Test Run Artifacts
        id: download-artifacts-from-vm
        if: always() && steps.spin-up-vm.outcome == 'success' && contains(fromJSON('["canceled", "skipped"]'), steps.run-tests.outcome) == false
        run: |
          tools --timestamps vm download-artifacts ${{ inputs.distro-slug }}
          tree -a artifacts
          mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ env.NOX_SESSION }}.${{ matrix.tests-chunk }}
          echo "COVERAGE_FILE=artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ env.NOX_SESSION }}.${{ matrix.tests-chunk }}" >> GITHUB_ENV

      - name: Destroy VM
        if: always()
        run: |
          tools --timestamps vm destroy ${{ inputs.distro-slug }} || true

      - name: Upload Test Run Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v3
        with:
          name: testrun-artifacts-${{ inputs.distro-slug }}-${{ env.NOX_SESSION }}
          path: artifacts

#      - name: Publish Test Report
#        uses: mikepenz/action-junit-report@v3
#        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
#        with:
#          check_name: Test Results(${{ inputs.distro-slug }} ${{ matrix.tests-chunk }})
#          report_paths: 'artifacts/xml-unittests-output/*.xml'
#          annotate_only: true

      - name: Install Nox
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        env:
          PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
          PIP_EXTRA_INDEX_URL: https://pypi.org/simple
        run: |
          python3 -m pip install 'nox==${{ env.NOX_VERSION }}'

      - name: Report Salt Code Coverage(${{ matrix.tests-chunk }})
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        continue-on-error: true
        run: |
          nox --force-color -e report-coverage -- salt

      - name: Report Tests Code Coverage(${{ matrix.tests-chunk }})
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        continue-on-error: true
        run: |
          nox --force-color -e report-coverage -- tests

      - name: Report Combined Code Coverage(${{ matrix.tests-chunk }})
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        continue-on-error: true
        run: |
          nox --force-color -e report-coverage

      - name: Set Exit Status
        if: always()
        run: |
          python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
          echo "${{ job.status }}" > exitstatus/${{ github.job }}-${{ inputs.distro-slug }}-${{ env.NOX_SESSION }}-${{ matrix.tests-chunk }}-tests

      - name: Upload Exit Status
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: exitstatus
          path: exitstatus
          if-no-files-found: error

  report:
    name: Reports for ${{ inputs.distro-slug }}(${{ matrix.transport }})
    runs-on:
      - self-hosted
      - linux
      - x86_64
    if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped'
    needs:
      - test
      - generate-matrix
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include) }}

    steps:
      - name: Checkout Source Code
        uses: actions/checkout@v3

      - name: Define Nox Session
        run: |
          if [ "${{ matrix.transport }}" != "tcp" ]; then
            echo NOX_SESSION=${{ inputs.nox-session }} >> $GITHUB_ENV
          else
            echo NOX_SESSION=${{ inputs.nox-session }}-tcp >> $GITHUB_ENV
          fi

      - name: Download Test Run Artifacts
        id: download-test-run-artifacts
        uses: actions/download-artifact@v3
        with:
          name: testrun-artifacts-${{ inputs.distro-slug }}-${{ env.NOX_SESSION }}
          path: artifacts

      - name: Show Test Run Artifacts
        if: always() && steps.download-test-run-artifacts.outcome == 'success'
        run: |
          tree -a artifacts

      - name: Upload Code Coverage DB
        if: always() && steps.download-test-run-artifacts.outcome == 'success'
        uses: actions/upload-artifact@v3
        with:
          name: code-coverage
          path: artifacts/coverage

      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - name: Install Nox
        run: |
          python3 -m pip install 'nox==${{ env.NOX_VERSION }}'

      - name: Report Salt Code Coverage
        continue-on-error: true
        run: |
          nox --force-color -e report-coverage -- salt

      - name: Report Tests Code Coverage
        continue-on-error: true
        run: |
          nox --force-color -e report-coverage -- tests

      - name: Report Combined Code Coverage
        continue-on-error: true
        run: |
          nox --force-color -e report-coverage

      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v3
        # always run even if the previous steps fails
        if: always() && github.event_name == 'push' && steps.download-test-run-artifacts.outcome == 'success'
        with:
          check_name: Overall Test Results(${{ inputs.distro-slug }})
          report_paths: 'artifacts/xml-unittests-output/*.xml'
          annotate_only: true

      - name: Set Exit Status
        if: always()
        run: |
          python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
          echo "${{ job.status }}" > exitstatus/${{ github.job }}-${{ inputs.distro-slug }}-${{ env.NOX_SESSION }}-report

      - name: Upload Exit Status
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: exitstatus
          path: exitstatus
          if-no-files-found: error