From 2712a0f863249b36040195f81f34c18be54afd1c Mon Sep 17 00:00:00 2001 From: "Daniel A. Wozniak" Date: Tue, 26 Nov 2024 17:04:27 -0700 Subject: [PATCH] Add mac and windows to tests action --- .github/workflows/test-action.yml | 476 ++++++++++++++++++++++++++++++ 1 file changed, 476 insertions(+) diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index 37960deeb49..18d9e3ce495 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -311,11 +311,487 @@ jobs: artifacts/logs include-hidden-files: true + test-macos: + name: ${{ matrix.display_name }} ${{ matrix.tests-chunk }} ${{ matrix.transport }} + runs-on: ${{ matrix.runner }} + # Full test runs. Each chunk should never take more than 2 hours. + # Partial test runs(no chunk parallelization), 6 Hours + timeout-minutes: ${{ fromJSON(inputs.testrun)['type'] == 'full' && inputs.default-timeout || 360 }} + needs: + - generate-matrix + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(inputs.matrix)['linux'] }} + env: + SALT_TRANSPORT: ${{ matrix.transport }} + + steps: + + - name: "Throttle Builds" + shell: bash + run: | + t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" + + - name: "Set `TIMESTAMP` environment variable" + shell: bash + run: | + echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV" + + - name: Checkout Source Code + uses: actions/checkout@v4 + + - name: Setup Salt Version + run: | + echo "${{ inputs.salt-version }}" > salt/_version.txt + + - name: Download Onedir Tarball as an Artifact + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.package-name }}-${{ inputs.salt-version }}-onedir-${{ matrix.platform }}-${{ matrix.arch }}.tar.xz + path: artifacts/ + + - name: Decompress Onedir Tarball + shell: bash + run: | + python3 -c "import os; os.makedirs('artifacts', exist_ok=True)" + cd artifacts + tar xvf ${{ matrix.package-name }}-${{ inputs.salt-version }}-onedir-${{ matrix.platform }}-${{ matrix.arch }}.tar.xz + + - name: Install System Dependencies + run: | + brew install tree + + - name: Download nox.macos.${{ matrix.arch }}.tar.* artifact for session ${{ inputs.nox-session }} + uses: actions/download-artifact@v4 + with: + name: nox-macos-${{ matrix.arch }}-${{ inputs.nox-session }} + + - name: Set up Python ${{ inputs.gh-actions-python-version }} + uses: actions/setup-python@v5 + with: + python-version: "${{ inputs.gh-actions-python-version }}" + + - name: Install Nox + run: | + python3 -m pip install 'nox==${{ inputs.nox-version }}' + env: + PIP_INDEX_URL: https://pypi.org/simple + + - name: Decompress .nox Directory + run: | + nox --force-color -e decompress-dependencies -- macos ${{ matrix.arch }} + + - name: Download testrun-changed-files.txt + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }} + uses: actions/download-artifact@v4 + with: + name: testrun-changed-files.txt + + - name: Show System Info + env: + SKIP_REQUIREMENTS_INSTALL: "1" + PRINT_SYSTEM_INFO_ONLY: "1" + run: | + sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} + + - name: Run Changed Tests + id: run-fast-changed-tests + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }} + env: + SKIP_REQUIREMENTS_INSTALL: "1" + PRINT_TEST_SELECTION: "0" + PRINT_TEST_PLAN_ONLY: "0" + PRINT_SYSTEM_INFO: "0" + RERUN_FAILURES: "1" + GITHUB_ACTIONS_PIPELINE: "1" + SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" + SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" + COVERAGE_CONTEXT: ${{ matrix.slug }} + run: | + sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ + -k "mac or darwin" --core-tests --slow-tests --suppress-no-test-exit-code \ + --from-filenames=testrun-changed-files.txt + + - name: Run Fast Tests + id: run-fast-tests + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }} + env: + SKIP_REQUIREMENTS_INSTALL: "1" + PRINT_TEST_SELECTION: "0" + PRINT_TEST_PLAN_ONLY: "0" + PRINT_SYSTEM_INFO: "0" + RERUN_FAILURES: "1" + GITHUB_ACTIONS_PIPELINE: "1" + SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" + SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" + COVERAGE_CONTEXT: ${{ matrix.slug }} + run: | + sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ + -k "mac or darwin" --suppress-no-test-exit-code + + - name: Run Slow Tests + id: run-slow-tests + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }} + env: + SKIP_REQUIREMENTS_INSTALL: "1" + PRINT_TEST_SELECTION: "0" + PRINT_TEST_PLAN_ONLY: "0" + PRINT_SYSTEM_INFO: "0" + RERUN_FAILURES: "1" + GITHUB_ACTIONS_PIPELINE: "1" + SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" + SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" + COVERAGE_CONTEXT: ${{ matrix.slug }} + run: | + sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ + -k "mac or darwin" --suppress-no-test-exit-code --no-fast-tests --slow-tests + + - name: Run Core Tests + id: run-core-tests + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }} + env: + SKIP_REQUIREMENTS_INSTALL: "1" + PRINT_TEST_SELECTION: "0" + PRINT_TEST_PLAN_ONLY: "0" + PRINT_SYSTEM_INFO: "0" + RERUN_FAILURES: "1" + GITHUB_ACTIONS_PIPELINE: "1" + SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" + SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" + COVERAGE_CONTEXT: ${{ matrix.slug }} + run: | + sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ + -k "mac or darwin" --suppress-no-test-exit-code --no-fast-tests --core-tests + + - name: Run Flaky Tests + id: run-flaky-tests + if: ${{ fromJSON(inputs.testrun)['selected_tests']['flaky'] }} + env: + SKIP_REQUIREMENTS_INSTALL: "1" + PRINT_TEST_SELECTION: "0" + PRINT_TEST_PLAN_ONLY: "0" + PRINT_SYSTEM_INFO: "0" + RERUN_FAILURES: "1" + GITHUB_ACTIONS_PIPELINE: "1" + SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" + SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" + COVERAGE_CONTEXT: ${{ matrix.slug }} + run: | + sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ + -k "mac or darwin" --suppress-no-test-exit-code --no-fast-tests --flaky-jail + + - name: Run Full Tests + id: run-full-tests + if: ${{ fromJSON(inputs.testrun)['type'] == 'full' }} + env: + SKIP_REQUIREMENTS_INSTALL: "1" + PRINT_TEST_SELECTION: "0" + PRINT_TEST_PLAN_ONLY: "0" + PRINT_SYSTEM_INFO: "0" + RERUN_FAILURES: "1" + GITHUB_ACTIONS_PIPELINE: "1" + SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" + SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" + COVERAGE_CONTEXT: ${{ matrix.slug }} + run: | + sudo -E nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- \ + --slow-tests --core-tests -k "mac or darwin" + + - name: Fix file ownership + run: | + sudo chown -R "$(id -un)" . + + - name: Combine Coverage Reports + if: always() && inputs.skip-code-coverage == false + run: | + nox --force-color -e combine-coverage + + - name: Prepare Test Run Artifacts + id: download-artifacts-from-vm + if: always() + run: | + # Delete the salt onedir, we won't need it anymore and it will prevent + # from it showing in the tree command below + rm -rf artifacts/salt* + tree -a artifacts + if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then + mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }} + fi + + - name: Upload Code Coverage Test Run Artifacts + if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' + uses: actions/upload-artifact@v4 + with: + name: testrun-coverage-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ env.TIMESTAMP }} + path: | + artifacts/coverage/ + include-hidden-files: true + + - name: Upload JUnit XML Test Run Artifacts + if: always() && steps.download-artifacts-from-vm.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: testrun-junit-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ env.TIMESTAMP }} + path: | + artifacts/xml-unittests-output/ + include-hidden-files: true + + - name: Upload Test Run Log Artifacts + if: always() && steps.download-artifacts-from-vm.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: testrun-log-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ env.TIMESTAMP }} + path: | + artifacts/logs + include-hidden-files: true + + test-windows: + name: ${{ matrix.display_name }} ${{ matrix.tests-chunk }} ${{ matrix.transport }} + runs-on: ${{ matrix.slug }} + # Full test runs. Each chunk should never take more than 2 hours. + # Partial test runs(no chunk parallelization), 6 Hours + timeout-minutes: ${{ fromJSON(inputs.testrun)['type'] == 'full' && inputs.default-timeout || 360 }} + needs: + - generate-matrix + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(inputs.matrix)['linux'] }} + env: + SALT_TRANSPORT: ${{ matrix.transport }} + + steps: + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: "Throttle Builds" + shell: bash + run: | + t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" + + - name: "Set `TIMESTAMP` environment variable" + shell: bash + run: | + echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV" + + - name: Checkout Source Code + uses: actions/checkout@v4 + + - name: Setup Salt Version + run: | + echo "${{ inputs.salt-version }}" > salt/_version.txt + + - name: Download Onedir Tarball as an Artifact + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.package-name }}-${{ inputs.salt-version }}-onedir-${{ matrix.platform }}-${{ matrix.arch }}.tar.xz + path: artifacts/ + + - name: Decompress Onedir Tarball + shell: bash + run: | + python3 -c "import os; os.makedirs('artifacts', exist_ok=True)" + cd artifacts + tar xvf ${{ matrix.package-name }}-${{ inputs.salt-version }}-onedir-${{ matrix.platform }}-${{ matrix.arch }}.tar.xz + + - name: Install System Dependencies + run: | + echo true + + - name: Set up Python ${{ inputs.gh-actions-python-version }} + uses: actions/setup-python@v5 + with: + python-version: "${{ inputs.gh-actions-python-version }}" + + - name: Install Nox + run: | + python3 -m pip install 'nox==${{ inputs.nox-version }}' + env: + PIP_INDEX_URL: https://pypi.org/simple + + - name: Download nox.windows.${{ matrix.arch }}.tar.* artifact for session ${{ inputs.nox-session }} + uses: actions/download-artifact@v4 + with: + name: nox-windows-${{ matrix.arch }}-${{ inputs.nox-session }} + + - name: Decompress .nox Directory + run: | + nox --force-color -e decompress-dependencies -- windows ${{ matrix.arch }} + + - name: Download testrun-changed-files.txt + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }} + uses: actions/download-artifact@v4 + with: + name: testrun-changed-files.txt + + - name: Check nox python + continue-on-error: true + run: | + .nox/ci-test-onedir/Scripts/python.exe --version + + - name: Show System Info + env: + SKIP_REQUIREMENTS_INSTALL: "1" + PRINT_SYSTEM_INFO_ONLY: "1" + run: | + nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} + + - name: Run Changed Tests + id: run-fast-changed-tests + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }} + env: + SKIP_REQUIREMENTS_INSTALL: "1" + PRINT_TEST_SELECTION: "0" + PRINT_TEST_PLAN_ONLY: "0" + PRINT_SYSTEM_INFO: "0" + RERUN_FAILURES: "1" + GITHUB_ACTIONS_PIPELINE: "1" + SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" + SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" + COVERAGE_CONTEXT: ${{ matrix.slug }} + run: > + nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- + -k "win" --core-tests --slow-tests --suppress-no-test-exit-code + "--from-filenames=testrun-changed-files.txt" + + - name: Run Fast Tests + id: run-fast-tests + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }} + env: + SKIP_REQUIREMENTS_INSTALL: "1" + PRINT_TEST_SELECTION: "0" + PRINT_TEST_PLAN_ONLY: "0" + PRINT_SYSTEM_INFO: "0" + RERUN_FAILURES: "1" + GITHUB_ACTIONS_PIPELINE: "1" + SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" + SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" + COVERAGE_CONTEXT: ${{ matrix.slug }} + run: > + nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- + -k "win" --suppress-no-test-exit-code + + - name: Run Slow Tests + id: run-slow-tests + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }} + env: + SKIP_REQUIREMENTS_INSTALL: "1" + PRINT_TEST_SELECTION: "0" + PRINT_TEST_PLAN_ONLY: "0" + PRINT_SYSTEM_INFO: "0" + RERUN_FAILURES: "1" + GITHUB_ACTIONS_PIPELINE: "1" + SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" + SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" + COVERAGE_CONTEXT: ${{ matrix.slug }} + run: > + nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- + -k "win" --suppress-no-test-exit-code --no-fast-tests --slow-tests + + - name: Run Core Tests + id: run-core-tests + if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }} + env: + SKIP_REQUIREMENTS_INSTALL: "1" + PRINT_TEST_SELECTION: "0" + PRINT_TEST_PLAN_ONLY: "0" + PRINT_SYSTEM_INFO: "0" + RERUN_FAILURES: "1" + GITHUB_ACTIONS_PIPELINE: "1" + SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" + SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" + COVERAGE_CONTEXT: ${{ matrix.slug }} + run: > + nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- + -k "win" --suppress-no-test-exit-code --no-fast-tests --core-tests + + - name: Run Flaky Tests + id: run-flaky-tests + if: ${{ fromJSON(inputs.testrun)['selected_tests']['flaky'] }} + env: + SKIP_REQUIREMENTS_INSTALL: "1" + PRINT_TEST_SELECTION: "0" + PRINT_TEST_PLAN_ONLY: "0" + PRINT_SYSTEM_INFO: "0" + RERUN_FAILURES: "1" + GITHUB_ACTIONS_PIPELINE: "1" + SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" + SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" + COVERAGE_CONTEXT: ${{ matrix.slug }} + run: > + nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- + -k "win" --suppress-no-test-exit-code --no-fast-tests --flaky-jail + + - name: Run Full Tests + id: run-full-tests + if: ${{ fromJSON(inputs.testrun)['type'] == 'full' }} + env: + SKIP_REQUIREMENTS_INSTALL: "1" + PRINT_TEST_SELECTION: "0" + PRINT_TEST_PLAN_ONLY: "0" + PRINT_SYSTEM_INFO: "0" + RERUN_FAILURES: "1" + GITHUB_ACTIONS_PIPELINE: "1" + SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" + SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" + COVERAGE_CONTEXT: ${{ matrix.slug }} + run: > + nox --force-color -e ${{ inputs.nox-session }} -- ${{ matrix.tests-chunk }} -- + --slow-tests --core-tests -k "win" + + - name: Combine Coverage Reports + if: always() && inputs.skip-code-coverage == false + run: | + nox --force-color -e combine-coverage + + - name: Prepare Test Run Artifacts + id: download-artifacts-from-vm + if: always() + shell: bash + run: | + # Delete the salt onedir, we won't need it anymore and it will prevent + # from it showing in the tree command below + rm -rf artifacts/salt* + if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then + mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ matrix.slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }} + fi + + - name: Upload Code Coverage Test Run Artifacts + if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' + uses: actions/upload-artifact@v4 + with: + name: testrun-coverage-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ env.TIMESTAMP }} + path: | + artifacts/coverage/ + include-hidden-files: true + + - name: Upload JUnit XML Test Run Artifacts + if: always() && steps.download-artifacts-from-vm.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: testrun-junit-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ env.TIMESTAMP }} + path: | + artifacts/xml-unittests-output/ + include-hidden-files: true + + - name: Upload Test Run Log Artifacts + if: always() && steps.download-artifacts-from-vm.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: testrun-log-artifacts-${{ matrix.slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-${{ env.TIMESTAMP }} + path: | + artifacts/logs + include-hidden-files: true + + report: name: Test Reports runs-on: ubuntu-latest needs: - test-linux + - test-macos strategy: fail-fast: false matrix: