diff --git a/.github/actions/download-artifact/action.yml b/.github/actions/download-artifact/action.yml deleted file mode 100644 index f1b8e547b14..00000000000 --- a/.github/actions/download-artifact/action.yml +++ /dev/null @@ -1,42 +0,0 @@ -# This actions was inspired by https://github.com/alehechka/download-tartifact ---- -name: Download Tar Artifact -description: > - Download and extract a tar artifact that was previously uploaded in the - workflow by the upload-tartifact action - -inputs: - name: - description: Artifact name - required: false - path: - description: Destination path - required: false - archive-name: - description: > - By default `inputs.name`(last resort, `archive`) is what's used to name the archive. - This parameter allows a customizing that archive name. This will allow uploading multiple - archives under the same 'name', like the underlying official action does - without overriding the existing archives. - required: false - - -runs: - using: composite - steps: - - uses: actions/download-artifact@v3 - # This needs to be actions/download-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. - with: - name: ${{ inputs.name }} - path: ${{ inputs.path }} - - - shell: bash - working-directory: ${{ inputs.path }} - run: | - tar -xvf ${{ inputs.archive-name || inputs.name || 'archive' }}.tar.gz - - - shell: bash - working-directory: ${{ inputs.path }} - run: | - rm -f ${{ inputs.archive-name || inputs.name || 'archive' }}.tar.gz diff --git a/.github/actions/upload-artifact/action.yml b/.github/actions/upload-artifact/action.yml index eb00196a339..012a2e80109 100644 --- a/.github/actions/upload-artifact/action.yml +++ b/.github/actions/upload-artifact/action.yml @@ -46,9 +46,7 @@ runs: shopt -s globstar || echo "'globstar' not available" tar -cavf ${{ inputs.archive-name || inputs.name || 'archive' }}.tar.gz ${{ inputs.path }} - - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + - uses: actions/upload-artifact@v4 with: name: ${{ inputs.name }} path: ${{ inputs.archive-name || inputs.name || 'archive' }}.tar.gz diff --git a/.github/workflows/build-deps-ci-action.yml b/.github/workflows/build-deps-ci-action.yml index 58ef83be4f1..73c328a97f7 100644 --- a/.github/workflows/build-deps-ci-action.yml +++ b/.github/workflows/build-deps-ci-action.yml @@ -47,8 +47,36 @@ env: jobs: + generate-matrix: + name: Generate Matrix + runs-on: ubuntu-latest + outputs: + matrix-include: ${{ steps.generate-matrix.outputs.matrix }} + steps: + + - name: "Throttle Builds" + shell: bash + run: | + t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t" + + - name: Checkout Source Code + uses: actions/checkout@v4 + + - name: Setup Python Tools Scripts + uses: ./.github/actions/setup-python-tools-scripts + with: + cache-prefix: ${{ inputs.cache-prefix }} + + - name: Generate Test Matrix + id: generate-matrix + run: | + tools ci deps-matrix + + linux-dependencies: name: Linux + needs: + - generate-matrix runs-on: - self-hosted - linux @@ -59,11 +87,7 @@ jobs: strategy: fail-fast: false matrix: - include: - - distro-slug: centos-7 - arch: x86_64 - - distro-slug: centos-7-arm64 - arch: arm64 + include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['linux'] }} steps: - name: "Throttle Builds" @@ -163,16 +187,14 @@ jobs: macos-dependencies: name: MacOS + needs: + - generate-matrix runs-on: ${{ matrix.distro-slug == 'macos-13-arm64' && 'macos-13-xlarge' || matrix.distro-slug }} timeout-minutes: 90 strategy: fail-fast: false matrix: - include: - - distro-slug: macos-12 - arch: x86_64 - - distro-slug: macos-13-arm64 - arch: arm64 + include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['macos'] }} steps: - name: "Throttle Builds" @@ -250,6 +272,8 @@ jobs: path: nox.macos.${{ matrix.arch }}.tar.* windows-dependencies: + needs: + - generate-matrix name: Windows runs-on: - self-hosted @@ -261,9 +285,7 @@ jobs: strategy: fail-fast: false matrix: - include: - - distro-slug: windows-2022 - arch: amd64 + include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['windows'] }} steps: - name: "Throttle Builds" diff --git a/.github/workflows/build-deps-onedir.yml b/.github/workflows/build-deps-onedir.yml index df4d699a87c..26a5812cc94 100644 --- a/.github/workflows/build-deps-onedir.yml +++ b/.github/workflows/build-deps-onedir.yml @@ -92,9 +92,7 @@ jobs: fail-fast: false max-parallel: 2 matrix: - arch: - - x86_64 - - arm64 + arch: ${{ github.event.repository.fork && fromJSON('["x86_64"]') || fromJSON('["x86_64", "arm64"]') }} runs-on: - ${{ matrix.arch == 'arm64' && 'macos-13-xlarge' || 'macos-12' }} env: diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml index 58b181eab1c..652bfde94fc 100644 --- a/.github/workflows/build-packages.yml +++ b/.github/workflows/build-packages.yml @@ -51,9 +51,7 @@ jobs: strategy: fail-fast: false matrix: - arch: - - x86_64 - - arm64 + arch: ${{ github.event.repository.fork && fromJSON('["x86_64"]') || fromJSON('["x86_64", "arm64"]') }} source: - ${{ inputs.source }} diff --git a/.github/workflows/build-salt-onedir.yml b/.github/workflows/build-salt-onedir.yml index ff4b7a3857f..73f9533fb51 100644 --- a/.github/workflows/build-salt-onedir.yml +++ b/.github/workflows/build-salt-onedir.yml @@ -99,9 +99,7 @@ jobs: fail-fast: false max-parallel: 2 matrix: - arch: - - x86_64 - - arm64 + arch: ${{ github.event.repository.fork && fromJSON('["x86_64"]') || fromJSON('["x86_64", "arm64"]') }} runs-on: - ${{ matrix.arch == 'arm64' && 'macos-13-xlarge' || 'macos-12' }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5818d2bd099..1073b1aceb8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1831,11 +1831,10 @@ jobs: - name: Get coverage reports id: get-coverage-reports - uses: actions/download-artifact@v3 - # This needs to be actions/download-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/download-artifact@v4 with: - name: all-testrun-coverage-artifacts + pattern: all-testrun-coverage-artifacts-* + merge-multiple: true path: artifacts/coverage/ - name: Display structure of downloaded files @@ -1997,12 +1996,12 @@ jobs: steps: - name: Get workflow information id: get-workflow-info - uses: technote-space/workflow-conclusion-action@v3 + uses: im-open/workflow-conclusion@v2 - name: Set Pipeline Exit Status shell: bash run: | - if [ "${{ steps.get-workflow-info.outputs.conclusion }}" != "success" ]; then + if [ "${{ steps.get-workflow-info.outputs.workflow_conclusion }}" != "success" ]; then exit 1 else exit 0 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5f60aa88dd8..76f4e68827e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1896,11 +1896,10 @@ jobs: - name: Get coverage reports id: get-coverage-reports - uses: actions/download-artifact@v3 - # This needs to be actions/download-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/download-artifact@v4 with: - name: all-testrun-coverage-artifacts + pattern: all-testrun-coverage-artifacts-* + merge-multiple: true path: artifacts/coverage/ - name: Display structure of downloaded files @@ -2080,7 +2079,7 @@ jobs: - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-nightly-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-nightly-repo-src path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error @@ -2211,7 +2210,7 @@ jobs: - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-nightly-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-nightly-repo-${{ matrix.pkg-type }}-${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }} path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error @@ -2401,7 +2400,7 @@ jobs: - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-nightly-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-nightly-repo-${{ matrix.pkg-type }}-${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }} path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error @@ -2550,6 +2549,7 @@ jobs: path: artifacts/pkgs/incoming - name: Download macOS Arch64 Packages + if: ${{ ! github.event.repository.fork }} uses: actions/download-artifact@v4 with: name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-arm64-macos @@ -2597,7 +2597,7 @@ jobs: - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-nightly-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-nightly-repo-macos path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error @@ -2654,6 +2654,7 @@ jobs: path: artifacts/pkgs/incoming - name: Download macOS arm64 Onedir Archive + if: ${{ ! github.event.repository.fork }} uses: actions/download-artifact@v4 with: name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-onedir-macos-arm64.tar.xz @@ -2725,7 +2726,7 @@ jobs: - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-nightly-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-nightly-repo-onedir path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error @@ -2796,11 +2797,10 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - name: Download Repository Artifact - uses: actions/download-artifact@v3 - # This needs to be actions/download-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/download-artifact@v4 with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-nightly-repo + pattern: salt-${{ needs.prepare-workflow.outputs.salt-version }}-nightly-repo-* + merge-multiple: true path: repo/ - name: Decompress Repository Artifacts @@ -2871,7 +2871,7 @@ jobs: steps: - name: Get workflow information id: get-workflow-info - uses: technote-space/workflow-conclusion-action@v3 + uses: im-open/workflow-conclusion@v2 - name: Notify Slack id: slack @@ -2935,7 +2935,7 @@ jobs: - name: Set Pipeline Exit Status shell: bash run: | - if [ "${{ steps.get-workflow-info.outputs.conclusion }}" != "success" ]; then + if [ "${{ steps.get-workflow-info.outputs.workflow_conclusion }}" != "success" ]; then exit 1 else exit 0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fac6c1cfc17..ff943fe4a17 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -461,7 +461,7 @@ jobs: steps: - name: Get workflow information id: get-workflow-info - uses: technote-space/workflow-conclusion-action@v3 + uses: im-open/workflow-conclusion@v2 - run: | # shellcheck disable=SC2129 @@ -475,7 +475,7 @@ jobs: - name: Set Pipeline Exit Status shell: bash run: | - if [ "${{ steps.get-workflow-info.outputs.conclusion }}" != "success" ]; then + if [ "${{ steps.get-workflow-info.outputs.workflow_conclusion }}" != "success" ]; then exit 1 else exit 0 diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 560a81abcb1..f6955a082d4 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -1878,11 +1878,10 @@ jobs: - name: Get coverage reports id: get-coverage-reports - uses: actions/download-artifact@v3 - # This needs to be actions/download-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/download-artifact@v4 with: - name: all-testrun-coverage-artifacts + pattern: all-testrun-coverage-artifacts-* + merge-multiple: true path: artifacts/coverage/ - name: Display structure of downloaded files @@ -2046,12 +2045,12 @@ jobs: steps: - name: Get workflow information id: get-workflow-info - uses: technote-space/workflow-conclusion-action@v3 + uses: im-open/workflow-conclusion@v2 - name: Set Pipeline Exit Status shell: bash run: | - if [ "${{ steps.get-workflow-info.outputs.conclusion }}" != "success" ]; then + if [ "${{ steps.get-workflow-info.outputs.workflow_conclusion }}" != "success" ]; then exit 1 else exit 0 diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index adc1384aa44..2ccd6a66d16 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -1916,7 +1916,7 @@ jobs: - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-repo-src path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error @@ -2047,7 +2047,7 @@ jobs: - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-repo-${{ matrix.pkg-type }}-${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }} path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error @@ -2239,7 +2239,7 @@ jobs: - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-repo-${{ matrix.pkg-type }}-${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }} path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error @@ -2388,6 +2388,7 @@ jobs: path: artifacts/pkgs/incoming - name: Download macOS Arch64 Packages + if: ${{ ! github.event.repository.fork }} uses: actions/download-artifact@v4 with: name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-arm64-macos @@ -2435,7 +2436,7 @@ jobs: - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-repo-macos path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error @@ -2492,6 +2493,7 @@ jobs: path: artifacts/pkgs/incoming - name: Download macOS arm64 Onedir Archive + if: ${{ ! github.event.repository.fork }} uses: actions/download-artifact@v4 with: name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-onedir-macos-arm64.tar.xz @@ -2563,7 +2565,7 @@ jobs: - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-repo-onedir path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error @@ -2603,11 +2605,10 @@ jobs: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - name: Download Repository Artifact - uses: actions/download-artifact@v3 - # This needs to be actions/download-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/download-artifact@v4 with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-repo + pattern: salt-${{ needs.prepare-workflow.outputs.salt-version }}-staging-repo-* + merge-multiple: true path: repo/ - name: Decompress Repository Artifacts @@ -2852,12 +2853,12 @@ jobs: steps: - name: Get workflow information id: get-workflow-info - uses: technote-space/workflow-conclusion-action@v3 + uses: im-open/workflow-conclusion@v2 - name: Set Pipeline Exit Status shell: bash run: | - if [ "${{ steps.get-workflow-info.outputs.conclusion }}" != "success" ]; then + if [ "${{ steps.get-workflow-info.outputs.workflow_conclusion }}" != "success" ]; then exit 1 else exit 0 diff --git a/.github/workflows/templates/build-deb-repo.yml.jinja b/.github/workflows/templates/build-deb-repo.yml.jinja index 73a5e47883c..91f8348385c 100644 --- a/.github/workflows/templates/build-deb-repo.yml.jinja +++ b/.github/workflows/templates/build-deb-repo.yml.jinja @@ -78,7 +78,7 @@ - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-repo-${{ matrix.pkg-type }}-${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }} path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/templates/build-deps-ci-action.yml.jinja b/.github/workflows/templates/build-deps-ci-action.yml.jinja deleted file mode 100644 index a08f02b0d3d..00000000000 --- a/.github/workflows/templates/build-deps-ci-action.yml.jinja +++ /dev/null @@ -1,364 +0,0 @@ ---- -name: Install Test Dependencies - -on: - workflow_call: - inputs: - nox-session: - required: true - type: string - description: The nox session to run - salt-version: - type: string - required: true - description: The Salt version to set prior to running tests. - cache-prefix: - required: true - type: string - description: Seed used to invalidate caches - nox-version: - required: true - type: string - description: The nox version to install - nox-archive-hash: - required: true - type: string - description: Nox Tarball Cache Hash - python-version: - required: false - type: string - description: The python version to run tests with - default: "3.10" - package-name: - required: false - type: string - description: The onedir package name to use - default: salt - - -env: - COLUMNS: 190 - AWS_MAX_ATTEMPTS: "10" - AWS_RETRY_MODE: "adaptive" - PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ - PIP_EXTRA_INDEX_URL: https://pypi.org/simple - PIP_DISABLE_PIP_VERSION_CHECK: "1" - RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" - -jobs: - - linux-dependencies: - name: Linux - runs-on: - - self-hosted - - linux - - bastion - env: - USE_S3_CACHE: 'true' - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - include: - <%- for arch, build_distro_slug in build_ci_deps_listing["linux"] %> - - distro-slug: <{ build_distro_slug }> - arch: <{ arch }> - <%- endfor %> - steps: - - - name: "Throttle Builds" - shell: bash - run: | - t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t" - - - name: Checkout Source Code - uses: actions/checkout@v4 - - - name: Cache nox.linux.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }} - id: nox-dependencies-cache - uses: ./.github/actions/cache - with: - path: nox.linux.${{ matrix.arch }}.tar.* - key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|linux|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }} - - - name: Download Onedir Tarball as an Artifact - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-linux-${{ matrix.arch }}.tar.xz - path: artifacts/ - - - name: Decompress Onedir Tarball - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - shell: bash - run: | - python3 -c "import os; os.makedirs('artifacts', exist_ok=True)" - cd artifacts - tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-linux-${{ matrix.arch }}.tar.xz - - - name: PyPi Proxy - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - sed -i '7s;^;--index-url=https://pypi-proxy.saltstack.net/root/local/+simple/ --extra-index-url=https://pypi.org/simple\n;' requirements/static/ci/*/*.txt - - - name: Setup Python Tools Scripts - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - uses: ./.github/actions/setup-python-tools-scripts - with: - cache-prefix: ${{ inputs.cache-prefix }}-build-deps-ci - - - name: Get Salt Project GitHub Actions Bot Environment - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") - SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) - echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" - - - name: Start VM - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - id: spin-up-vm - run: | - tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ matrix.distro-slug }} - - - name: List Free Space - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - tools --timestamps vm ssh ${{ matrix.distro-slug }} -- df -h || true - - - name: Upload Checkout To VM - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - tools --timestamps vm rsync ${{ matrix.distro-slug }} - - - name: Install Dependencies - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - tools --timestamps vm install-dependencies --nox-session=${{ inputs.nox-session }} ${{ matrix.distro-slug }} - - - name: Cleanup .nox Directory - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - tools --timestamps vm pre-archive-cleanup ${{ matrix.distro-slug }} - - - name: Compress .nox Directory - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - tools --timestamps vm compress-dependencies ${{ matrix.distro-slug }} - - - name: Download Compressed .nox Directory - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - tools --timestamps vm download-dependencies ${{ matrix.distro-slug }} - - - name: Destroy VM - if: always() && steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - tools --timestamps vm destroy --no-wait ${{ matrix.distro-slug }} - - - name: Upload Nox Requirements Tarball - uses: actions/upload-artifact@v4 - with: - name: nox-linux-${{ matrix.arch }}-${{ inputs.nox-session }} - path: nox.linux.${{ matrix.arch }}.tar.* - - macos-dependencies: - name: MacOS - runs-on: ${{ matrix.distro-slug == 'macos-13-arm64' && 'macos-13-xlarge' || matrix.distro-slug }} - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - include: - <%- for arch, build_distro_slug in build_ci_deps_listing["macos"] %> - - distro-slug: <{ build_distro_slug }> - arch: <{ arch }> - <%- endfor %> - steps: - - - name: "Throttle Builds" - shell: bash - run: | - t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" - - - name: Checkout Source Code - uses: actions/checkout@v4 - - - name: Cache nox.macos.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }} - id: nox-dependencies-cache - uses: ./.github/actions/cache - with: - path: nox.macos.${{ matrix.arch }}.tar.* - key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|macos|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }} - - - name: Download Onedir Tarball as an Artifact - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-macos-${{ matrix.arch }}.tar.xz - path: artifacts/ - - - name: Decompress Onedir Tarball - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - shell: bash - run: | - python3 -c "import os; os.makedirs('artifacts', exist_ok=True)" - cd artifacts - tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-macos-${{ matrix.arch }}.tar.xz - - - name: Set up Python ${{ inputs.python-version }} - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - uses: actions/setup-python@v5 - with: - python-version: "${{ inputs.python-version }}" - - - name: Install System Dependencies - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - brew install openssl@3 - - - name: Install Nox - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - python3 -m pip install 'nox==${{ inputs.nox-version }}' - - - name: Install Dependencies - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - env: - PRINT_TEST_SELECTION: "0" - PRINT_SYSTEM_INFO: "0" - run: | - export PYCURL_SSL_LIBRARY=openssl - export LDFLAGS="-L/usr/local/opt/openssl@3/lib" - export CPPFLAGS="-I/usr/local/opt/openssl@3/include" - export PKG_CONFIG_PATH="/usr/local/opt/openssl@3/lib/pkgconfig" - nox --install-only -e ${{ inputs.nox-session }} - - - name: Cleanup .nox Directory - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - nox --force-color -e "pre-archive-cleanup(pkg=False)" - - - name: Compress .nox Directory - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - nox --force-color -e compress-dependencies -- macos ${{ matrix.arch }} - - - name: Upload Nox Requirements Tarball - uses: actions/upload-artifact@v4 - with: - name: nox-macos-${{ matrix.arch }}-${{ inputs.nox-session }} - path: nox.macos.${{ matrix.arch }}.tar.* - - windows-dependencies: - name: Windows - runs-on: - - self-hosted - - linux - - bastion - env: - USE_S3_CACHE: 'true' - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - include: - <%- for arch, build_distro_slug in build_ci_deps_listing["windows"] %> - - distro-slug: <{ build_distro_slug }> - arch: <{ arch }> - <%- endfor %> - steps: - - - name: "Throttle Builds" - shell: bash - run: | - t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t" - - - name: Checkout Source Code - uses: actions/checkout@v4 - - - name: Cache nox.windows.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }} - id: nox-dependencies-cache - uses: ./.github/actions/cache - with: - path: nox.windows.${{ matrix.arch }}.tar.* - key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|windows|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }} - - - name: Download Onedir Tarball as an Artifact - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-windows-${{ matrix.arch }}.tar.xz - path: artifacts/ - - - name: Decompress Onedir Tarball - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - shell: bash - run: | - python3 -c "import os; os.makedirs('artifacts', exist_ok=True)" - cd artifacts - tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-windows-${{ matrix.arch }}.tar.xz - - - name: PyPi Proxy - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - sed -i '7s;^;--index-url=https://pypi-proxy.saltstack.net/root/local/+simple/ --extra-index-url=https://pypi.org/simple\n;' requirements/static/ci/*/*.txt - - - name: Setup Python Tools Scripts - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - uses: ./.github/actions/setup-python-tools-scripts - with: - cache-prefix: ${{ inputs.cache-prefix }}-build-deps-ci - - - name: Get Salt Project GitHub Actions Bot Environment - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") - SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) - echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" - - - name: Start VM - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - id: spin-up-vm - run: | - tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ matrix.distro-slug }} - - - name: List Free Space - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - tools --timestamps vm ssh ${{ matrix.distro-slug }} -- df -h || true - - - name: Upload Checkout To VM - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - tools --timestamps vm rsync ${{ matrix.distro-slug }} - - - name: Install Dependencies - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - tools --timestamps vm install-dependencies --nox-session=${{ inputs.nox-session }} ${{ matrix.distro-slug }} - - - name: Cleanup .nox Directory - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - tools --timestamps vm pre-archive-cleanup ${{ matrix.distro-slug }} - - - name: Compress .nox Directory - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - tools --timestamps vm compress-dependencies ${{ matrix.distro-slug }} - - - name: Download Compressed .nox Directory - if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - tools --timestamps vm download-dependencies ${{ matrix.distro-slug }} - - - name: Destroy VM - if: always() && steps.nox-dependencies-cache.outputs.cache-hit != 'true' - run: | - tools --timestamps vm destroy --no-wait ${{ matrix.distro-slug }} - - - name: Upload Nox Requirements Tarball - uses: actions/upload-artifact@v4 - with: - name: nox-windows-${{ matrix.arch }}-${{ inputs.nox-session }} - path: nox.windows.${{ matrix.arch }}.tar.* diff --git a/.github/workflows/templates/build-macos-repo.yml.jinja b/.github/workflows/templates/build-macos-repo.yml.jinja index e64297f5aad..835e366bf52 100644 --- a/.github/workflows/templates/build-macos-repo.yml.jinja +++ b/.github/workflows/templates/build-macos-repo.yml.jinja @@ -26,6 +26,7 @@ path: artifacts/pkgs/incoming - name: Download macOS Arch64 Packages + if: ${{ ! github.event.repository.fork }} uses: actions/download-artifact@v4 with: name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-arm64-macos @@ -73,7 +74,7 @@ - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-repo-macos path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/templates/build-onedir-repo.yml.jinja b/.github/workflows/templates/build-onedir-repo.yml.jinja index 8dada6d2ffc..c6b51f07166 100644 --- a/.github/workflows/templates/build-onedir-repo.yml.jinja +++ b/.github/workflows/templates/build-onedir-repo.yml.jinja @@ -38,6 +38,7 @@ path: artifacts/pkgs/incoming - name: Download macOS arm64 Onedir Archive + if: ${{ ! github.event.repository.fork }} uses: actions/download-artifact@v4 with: name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-onedir-macos-arm64.tar.xz @@ -109,7 +110,7 @@ - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-repo-onedir path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/templates/build-rpm-repo.yml.jinja b/.github/workflows/templates/build-rpm-repo.yml.jinja index f65755c33d8..7039043d4bf 100644 --- a/.github/workflows/templates/build-rpm-repo.yml.jinja +++ b/.github/workflows/templates/build-rpm-repo.yml.jinja @@ -85,7 +85,7 @@ - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-repo-${{ matrix.pkg-type }}-${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }} path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/templates/build-src-repo.yml.jinja b/.github/workflows/templates/build-src-repo.yml.jinja index 071430a24f6..8409f05879b 100644 --- a/.github/workflows/templates/build-src-repo.yml.jinja +++ b/.github/workflows/templates/build-src-repo.yml.jinja @@ -83,7 +83,7 @@ - name: Upload Repository As An Artifact uses: ./.github/actions/upload-artifact with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-repo + name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-repo-src path: artifacts/pkgs/repo/* retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index 6d753588059..09c0a6cfcda 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -349,11 +349,10 @@ - name: Get coverage reports id: get-coverage-reports - uses: actions/download-artifact@v3 - # This needs to be actions/download-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/download-artifact@v4 with: - name: all-testrun-coverage-artifacts + pattern: all-testrun-coverage-artifacts-* + merge-multiple: true path: artifacts/coverage/ - name: Display structure of downloaded files diff --git a/.github/workflows/templates/layout.yml.jinja b/.github/workflows/templates/layout.yml.jinja index a0ac92060a9..2efbee7bee6 100644 --- a/.github/workflows/templates/layout.yml.jinja +++ b/.github/workflows/templates/layout.yml.jinja @@ -358,7 +358,7 @@ jobs: steps: - name: Get workflow information id: get-workflow-info - uses: technote-space/workflow-conclusion-action@v3 + uses: im-open/workflow-conclusion@v2 <%- block set_pipeline_exit_status_extra_steps %> <%- endblock set_pipeline_exit_status_extra_steps %> @@ -366,7 +366,7 @@ jobs: - name: Set Pipeline Exit Status shell: bash run: | - if [ "${{ steps.get-workflow-info.outputs.conclusion }}" != "success" ]; then + if [ "${{ steps.get-workflow-info.outputs.workflow_conclusion }}" != "success" ]; then exit 1 else exit 0 diff --git a/.github/workflows/templates/nightly.yml.jinja b/.github/workflows/templates/nightly.yml.jinja index 313e7297150..d51582bc3af 100644 --- a/.github/workflows/templates/nightly.yml.jinja +++ b/.github/workflows/templates/nightly.yml.jinja @@ -172,11 +172,10 @@ concurrency: cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }} - name: Download Repository Artifact - uses: actions/download-artifact@v3 - # This needs to be actions/download-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/download-artifact@v4 with: - name: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-repo + pattern: salt-${{ needs.prepare-workflow.outputs.salt-version }}-<{ gh_environment }>-repo-* + merge-multiple: true path: repo/ - name: Decompress Repository Artifacts diff --git a/.github/workflows/templates/test-package-downloads-action.yml.jinja b/.github/workflows/templates/test-package-downloads-action.yml.jinja deleted file mode 100644 index 469509e979f..00000000000 --- a/.github/workflows/templates/test-package-downloads-action.yml.jinja +++ /dev/null @@ -1,679 +0,0 @@ -name: Test Download Packages - -on: - workflow_call: - inputs: - salt-version: - type: string - required: true - description: The Salt version of the packages to install and test - cache-prefix: - required: true - type: string - description: Seed used to invalidate caches - environment: - required: true - type: string - description: The environment to run tests against - latest-release: - required: true - type: string - description: The latest salt release - nox-version: - required: true - type: string - description: The nox version to install - python-version: - required: false - type: string - description: The python version to run tests with - default: "3.10" - package-name: - required: false - type: string - description: The onedir package name to use - default: salt - skip-code-coverage: - required: false - type: boolean - description: Skip code coverage - default: false - nox-session: - required: false - type: string - description: The nox session to run - default: ci-test-onedir - -env: - COLUMNS: 190 - AWS_MAX_ATTEMPTS: "10" - AWS_RETRY_MODE: "adaptive" - PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ - PIP_EXTRA_INDEX_URL: https://pypi.org/simple - PIP_DISABLE_PIP_VERSION_CHECK: "1" - RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" - -jobs: - - linux: - name: Linux - runs-on: - - self-hosted - - linux - - bastion - env: - USE_S3_CACHE: 'true' - environment: ${{ inputs.environment }} - timeout-minutes: 120 # 2 Hours - More than this and something is wrong - strategy: - fail-fast: false - matrix: - include: - <%- for os in test_salt_pkg_downloads_listing["linux"] %> - - distro-slug: <{ os.slug }> - arch: <{ os.arch }> - pkg-type: <{ os.pkg_type }> - <%- endfor %> - - steps: - - - name: "Throttle Builds" - shell: bash - run: | - t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t" - - - name: Checkout Source Code - uses: actions/checkout@v4 - - - name: Download Onedir Tarball as an Artifact - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-linux-${{ matrix.arch == 'aarch64' && 'arm64' || matrix.arch }}.tar.xz - path: artifacts/ - - - name: Decompress Onedir Tarball - shell: bash - run: | - python3 -c "import os; os.makedirs('artifacts', exist_ok=True)" - cd artifacts - tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-linux-${{ matrix.arch == 'aarch64' && 'arm64' || matrix.arch }}.tar.xz - - - name: Download nox.linux.${{ matrix.arch == 'aarch64' && 'arm64' || matrix.arch }}.tar.* artifact for session ${{ inputs.nox-session }} - uses: actions/download-artifact@v4 - with: - name: nox-linux-${{ matrix.arch == 'aarch64' && 'arm64' || matrix.arch }}-${{ inputs.nox-session }} - - - name: Setup Python Tools Scripts - uses: ./.github/actions/setup-python-tools-scripts - with: - cache-prefix: ${{ inputs.cache-prefix }}-pkg-download-linux - - - name: Get Salt Project GitHub Actions Bot Environment - run: | - TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") - SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) - echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" - - - name: Start VM - id: spin-up-vm - run: | - tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ matrix.distro-slug }} - - - name: List Free Space - run: | - tools --timestamps vm ssh ${{ matrix.distro-slug }} -- df -h || true - - - name: Upload Checkout To VM - run: | - tools --timestamps vm rsync ${{ matrix.distro-slug }} - - - name: Decompress .nox Directory - run: | - tools --timestamps vm decompress-dependencies ${{ matrix.distro-slug }} - - - name: Show System Info - run: | - tools --timestamps --timeout-secs=1800 vm test --skip-requirements-install --print-system-information-only \ - --nox-session=${{ inputs.nox-session }}-pkgs ${{ matrix.distro-slug }} -- download-pkgs - - - name: Run Package Download Tests - env: - SALT_RELEASE: "${{ inputs.salt-version }}" - SALT_REPO_ARCH: ${{ matrix.arch }} - SALT_REPO_TYPE: ${{ inputs.environment }} - SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }} - SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }} - SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }} - SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }} - SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" - LATEST_SALT_RELEASE: "${{ inputs.latest-release }}" - DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }} - run: | - tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \ - -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \ - -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE -E DOWNLOAD_TEST_PACKAGE_TYPE \ - --nox-session=${{ inputs.nox-session }}-pkgs --rerun-failures ${{ matrix.distro-slug }} -- download-pkgs - - - name: Combine Coverage Reports - if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' - run: | - tools --timestamps vm combine-coverage ${{ matrix.distro-slug }} - - - name: Download Test Run Artifacts - id: download-artifacts-from-vm - if: always() && steps.spin-up-vm.outcome == 'success' - run: | - tools --timestamps vm download-artifacts ${{ matrix.distro-slug }} - # Delete the salt onedir, we won't need it anymore and it will prevent - # from it showing in the tree command below - rm -rf artifacts/salt* - tree -a artifacts - - - name: Destroy VM - if: always() - run: | - tools --timestamps vm destroy --no-wait ${{ matrix.distro-slug }} || true - - - name: Fix file ownership - run: | - sudo chown -R "$(id -un)" . - - - name: Install Codecov CLI - if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' - run: | - # We can't yet use tokenless uploads with the codecov CLI - # python3 -m pip install codecov-cli - # - curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --import - curl -Os https://uploader.codecov.io/latest/linux/codecov - curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM - curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig - gpg --verify codecov.SHA256SUM.sig codecov.SHA256SUM - shasum -a 256 -c codecov.SHA256SUM - chmod +x codecov - - - name: Upload Source Code Coverage To Codecov - if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' - run: | - if [ ! -s artifacts/coverage/salt.xml ]; then - echo "The artifacts/coverage/salt.xml file does not exist" - exit 1 - fi - # We can't yet use tokenless uploads with the codecov CLI - #codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \ - # do-upload --git-service github --sha ${{ github.sha }} \ - # --file artifacts/coverage/salt.xml \ - # --flag salt --flag ${{ matrix.distro-slug }} --flag pkg \ - # --name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs - n=0 - until [ "$n" -ge 5 ] - do - if ./codecov --file artifacts/coverage/salt.xml \ - --sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \ - --flags salt,${{ matrix.distro-slug }},pkg \ - --name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then - rc=$? - break - fi - rc=$? - n=$((n+1)) - sleep 15 - done - if [ "$rc" -ne 0 ]; then - echo "Failed to upload codecov stats" - exit 1 - fi - - - name: Upload Tests Code Coverage To Codecov - if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' - run: | - if [ ! -s artifacts/coverage/tests.xml ]; then - echo "The artifacts/coverage/tests.xml file does not exist" - exit 1 - fi - # We can't yet use tokenless uploads with the codecov CLI - #codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \ - # do-upload --git-service github --sha ${{ github.sha }} \ - # --file artifacts/coverage/tests.xml \ - # --flag tests --flag ${{ matrix.distro-slug }} --flag pkg \ - # --name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs - n=0 - until [ "$n" -ge 5 ] - do - if ./codecov --file artifacts/coverage/tests.xml \ - --sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \ - --flags tests,${{ matrix.distro-slug }},pkg \ - --name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then - rc=$? - break - fi - rc=$? - n=$((n+1)) - sleep 15 - done - if [ "$rc" -ne 0 ]; then - echo "Failed to upload codecov stats" - exit 1 - fi - - - name: Upload Test Run Artifacts - if: always() && steps.download-artifacts-from-vm.outcome == 'success' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. - with: - name: pkg-testrun-artifacts-${{ matrix.distro-slug }}-${{ matrix.arch }} - path: | - artifacts - !artifacts/salt/* - !artifacts/salt-*.tar.* - - - macos: - name: MacOS - runs-on: ${{ matrix.distro-slug == 'macos-13-arm64' && 'macos-13-xlarge' || matrix.distro-slug }} - env: - USE_S3_CACHE: 'false' - environment: ${{ inputs.environment }} - timeout-minutes: 120 # 2 Hours - More than this and something is wrong - strategy: - fail-fast: false - matrix: - include: - <%- for os in test_salt_pkg_downloads_listing["macos"] %> - - distro-slug: <{ os.slug }> - arch: <{ os.arch }> - pkg-type: <{ os.pkg_type }> - <%- endfor %> - - steps: - - - name: "Throttle Builds" - shell: bash - run: | - t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" - - - name: Checkout Source Code - uses: actions/checkout@v4 - - - name: Download Onedir Tarball as an Artifact - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-macos-${{ matrix.arch }}.tar.xz - path: artifacts/ - - - name: Install System Dependencies - run: | - brew install tree - - - name: Decompress Onedir Tarball - shell: bash - run: | - python3 -c "import os; os.makedirs('artifacts', exist_ok=True)" - cd artifacts - tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-macos-${{ matrix.arch }}.tar.xz - - - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v5 - with: - python-version: "${{ inputs.python-version }}" - update-environment: true - - - name: Install Nox - run: | - python3 -m pip install 'nox==${{ inputs.nox-version }}' - - - name: Download nox.macos.${{ matrix.arch }}.tar.* artifact for session ${{ inputs.nox-session }} - uses: actions/download-artifact@v4 - with: - name: nox-macos-${{ matrix.arch }}-${{ inputs.nox-session }} - - - name: Decompress .nox Directory - run: | - nox --force-color -e decompress-dependencies -- macos ${{ matrix.arch }} - - - name: Show System Info - env: - SKIP_REQUIREMENTS_INSTALL: "1" - PRINT_SYSTEM_INFO_ONLY: "1" - run: | - sudo -E nox --force-color -e ${{ inputs.nox-session }}-pkgs -- download-pkgs - - - name: Run Package Download Tests - env: - SKIP_REQUIREMENTS_INSTALL: "1" - PRINT_TEST_SELECTION: "0" - PRINT_TEST_PLAN_ONLY: "0" - PRINT_SYSTEM_INFO: "0" - RERUN_FAILURES: "1" - GITHUB_ACTIONS_PIPELINE: "1" - SKIP_INITIAL_GH_ACTIONS_FAILURES: "1" - SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" - COVERAGE_CONTEXT: ${{ matrix.distro-slug }} - SALT_RELEASE: "${{ inputs.salt-version }}" - SALT_REPO_ARCH: ${{ matrix.arch }} - LATEST_SALT_RELEASE: "${{ inputs.latest-release }}" - SALT_REPO_TYPE: ${{ inputs.environment }} - SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }} - SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }} - SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }} - SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }} - DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }} - run: | - sudo -E nox --force-color -e ${{ inputs.nox-session }}-pkgs -- download-pkgs - - - name: Fix file ownership - run: | - sudo chown -R "$(id -un)" . - - - name: Combine Coverage Reports - if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled' - run: | - nox --force-color -e combine-coverage - - - name: Prepare Test Run Artifacts - id: download-artifacts-from-vm - if: always() && job.status != 'cancelled' - run: | - # Delete the salt onedir, we won't need it anymore and it will prevent - # from it showing in the tree command below - rm -rf artifacts/salt* - tree -a artifacts - - - name: Install Codecov CLI - if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled' - run: | - # We can't yet use tokenless uploads with the codecov CLI - # python3 -m pip install codecov-cli - # - curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --import - curl -Os https://uploader.codecov.io/latest/macos/codecov - curl -Os https://uploader.codecov.io/latest/macos/codecov.SHA256SUM - curl -Os https://uploader.codecov.io/latest/macos/codecov.SHA256SUM.sig - gpg --verify codecov.SHA256SUM.sig codecov.SHA256SUM - shasum -a 256 -c codecov.SHA256SUM - chmod +x codecov - - - name: Upload Source Code Coverage To Codecov - if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled' - run: | - if [ ! -s artifacts/coverage/salt.xml ]; then - echo "The artifacts/coverage/salt.xml file does not exist" - exit 1 - fi - # We can't yet use tokenless uploads with the codecov CLI - #codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \ - # do-upload --git-service github --sha ${{ github.sha }} \ - # --file artifacts/coverage/salt.xml \ - # --flag salt --flag ${{ matrix.distro-slug }} --flag pkg \ - # --name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs - n=0 - until [ "$n" -ge 5 ] - do - if ./codecov --file artifacts/coverage/salt.xml \ - --sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \ - --flags salt,${{ matrix.distro-slug }},pkg \ - --name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then - rc=$? - break - fi - rc=$? - n=$((n+1)) - sleep 15 - done - if [ "$rc" -ne 0 ]; then - echo "Failed to upload codecov stats" - exit 1 - fi - - - name: Upload Tests Code Coverage To Codecov - if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled' - run: | - if [ ! -s artifacts/coverage/tests.xml ]; then - echo "The artifacts/coverage/tests.xml file does not exist" - exit 1 - fi - # We can't yet use tokenless uploads with the codecov CLI - #codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \ - # do-upload --git-service github --sha ${{ github.sha }} \ - # --file artifacts/coverage/tests.xml \ - # --flag tests --flag ${{ matrix.distro-slug }} --flag pkg \ - # --name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs - n=0 - until [ "$n" -ge 5 ] - do - if ./codecov --file artifacts/coverage/tests.xml \ - --sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \ - --flags tests,${{ matrix.distro-slug }},pkg \ - --name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then - rc=$? - break - fi - rc=$? - n=$((n+1)) - sleep 15 - done - if [ "$rc" -ne 0 ]; then - echo "Failed to upload codecov stats" - exit 1 - fi - - - name: Upload Test Run Artifacts - if: always() - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. - with: - name: pkg-testrun-artifacts-${{ matrix.distro-slug }}-${{ matrix.arch }} - path: | - artifacts - !artifacts/salt/* - !artifacts/salt-*.tar.* - - - windows: - name: Windows - env: - USE_S3_CACHE: 'true' - runs-on: - - self-hosted - - linux - - bastion - environment: ${{ inputs.environment }} - timeout-minutes: 120 # 2 Hours - More than this and something is wrong - strategy: - fail-fast: false - matrix: - include: - <%- for os in test_salt_pkg_downloads_listing["windows"] %> - - distro-slug: <{ os.slug }> - arch: <{ os.arch }> - pkg-type: <{ os.pkg_type }> - <%- endfor %> - - steps: - - name: Checkout Source Code - uses: actions/checkout@v4 - - - name: Download Onedir Tarball as an Artifact - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-windows-${{ matrix.arch }}.tar.xz - path: artifacts/ - - - name: Decompress Onedir Tarball - shell: bash - run: | - python3 -c "import os; os.makedirs('artifacts', exist_ok=True)" - cd artifacts - tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-windows-${{ matrix.arch }}.tar.xz - - - name: Download nox.windows.${{ matrix.arch }}.tar.* artifact for session ${{ inputs.nox-session }} - uses: actions/download-artifact@v4 - with: - name: nox-windows-${{ matrix.arch }}-${{ inputs.nox-session }} - - - name: Setup Python Tools Scripts - uses: ./.github/actions/setup-python-tools-scripts - with: - cache-prefix: ${{ inputs.cache-prefix }}-pkg-download-windows - - - name: Get Salt Project GitHub Actions Bot Environment - run: | - TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") - SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) - echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" - - - name: Start VM - id: spin-up-vm - run: | - tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ matrix.distro-slug }} - - - name: List Free Space - run: | - tools --timestamps vm ssh ${{ matrix.distro-slug }} -- df -h || true - - - name: Upload Checkout To VM - run: | - tools --timestamps vm rsync ${{ matrix.distro-slug }} - - - name: Decompress .nox Directory - run: | - tools --timestamps vm decompress-dependencies ${{ matrix.distro-slug }} - - - name: Show System Info - run: | - tools --timestamps --timeout-secs=1800 vm test --skip-requirements-install --print-system-information-only \ - --nox-session=${{ inputs.nox-session }}-pkgs ${{ matrix.distro-slug }} -- download-pkgs - - - name: Run Package Download Tests - env: - SALT_RELEASE: "${{ inputs.salt-version }}" - SALT_REPO_ARCH: ${{ matrix.arch }} - LATEST_SALT_RELEASE: "${{ inputs.latest-release }}" - SALT_REPO_TYPE: ${{ inputs.environment }} - SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }} - SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }} - SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }} - SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }} - SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}" - DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }} - run: | - tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \ - -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \ - -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE -E DOWNLOAD_TEST_PACKAGE_TYPE \ - --nox-session=${{ inputs.nox-session }}-pkgs --rerun-failures ${{ matrix.distro-slug }} -- download-pkgs - - - name: Combine Coverage Reports - if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' - run: | - tools --timestamps vm combine-coverage ${{ matrix.distro-slug }} - - - name: Download Test Run Artifacts - id: download-artifacts-from-vm - if: always() && steps.spin-up-vm.outcome == 'success' - run: | - tools --timestamps vm download-artifacts ${{ matrix.distro-slug }} - # Delete the salt onedir, we won't need it anymore and it will prevent - # from it showing in the tree command below - rm -rf artifacts/salt* - tree -a artifacts - - - name: Destroy VM - if: always() - run: | - tools --timestamps vm destroy --no-wait ${{ matrix.distro-slug }} || true - - - name: Fix file ownership - run: | - sudo chown -R "$(id -un)" . - - - name: Install Codecov CLI - if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' - run: | - # We can't yet use tokenless uploads with the codecov CLI - # python3 -m pip install codecov-cli - # - curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --import - curl -Os https://uploader.codecov.io/latest/linux/codecov - curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM - curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig - gpg --verify codecov.SHA256SUM.sig codecov.SHA256SUM - shasum -a 256 -c codecov.SHA256SUM - chmod +x codecov - - - name: Upload Source Code Coverage To Codecov - if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' - run: | - if [ ! -s artifacts/coverage/salt.xml ]; then - echo "The artifacts/coverage/salt.xml file does not exist" - exit 1 - fi - # We can't yet use tokenless uploads with the codecov CLI - #codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \ - # do-upload --git-service github --sha ${{ github.sha }} \ - # --file artifacts/coverage/salt.xml \ - # --flag salt --flag ${{ matrix.distro-slug }} --flag pkg \ - # --name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs - n=0 - until [ "$n" -ge 5 ] - do - if ./codecov --file artifacts/coverage/salt.xml \ - --sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \ - --flags salt,${{ matrix.distro-slug }},pkg \ - --name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then - rc=$? - break - fi - rc=$? - n=$((n+1)) - sleep 15 - done - if [ "$rc" -ne 0 ]; then - echo "Failed to upload codecov stats" - exit 1 - fi - - - name: Upload Tests Code Coverage To Codecov - if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' - run: | - if [ ! -s artifacts/coverage/tests.xml ]; then - echo "The artifacts/coverage/tests.xml file does not exist" - exit 1 - fi - # We can't yet use tokenless uploads with the codecov CLI - #codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \ - # do-upload --git-service github --sha ${{ github.sha }} \ - # --file artifacts/coverage/tests.xml \ - # --flag tests --flag ${{ matrix.distro-slug }} --flag pkg \ - # --name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs - n=0 - until [ "$n" -ge 5 ] - do - if ./codecov --file artifacts/coverage/tests.xml \ - --sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \ - --flags tests,${{ matrix.distro-slug }},pkg \ - --name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then - rc=$? - break - fi - rc=$? - n=$((n+1)) - sleep 15 - done - if [ "$rc" -ne 0 ]; then - echo "Failed to upload codecov stats" - exit 1 - fi - - - name: Upload Test Run Artifacts - if: always() && steps.download-artifacts-from-vm.outcome == 'success' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. - with: - name: pkg-testrun-artifacts-${{ matrix.distro-slug }}-${{ matrix.arch }} - path: | - artifacts - !artifacts/salt/* - !artifacts/salt-*.tar.* diff --git a/.github/workflows/templates/test-salt-pkg-repo-downloads.yml.jinja b/.github/workflows/templates/test-salt-pkg-repo-downloads.yml.jinja index d28614620db..d547bd504db 100644 --- a/.github/workflows/templates/test-salt-pkg-repo-downloads.yml.jinja +++ b/.github/workflows/templates/test-salt-pkg-repo-downloads.yml.jinja @@ -13,9 +13,7 @@ needs: - prepare-workflow - publish-repositories - <%- for slug in test_salt_pkg_downloads_needs_slugs %> - - <{ slug }> - <%- endfor %> + - build-ci-deps <%- if gh_environment == "release" %> - download-onedir-artifact <%- else %> diff --git a/.github/workflows/test-action-linux.yml b/.github/workflows/test-action-linux.yml index 8cce5be2fad..49a33daeb5a 100644 --- a/.github/workflows/test-action-linux.yml +++ b/.github/workflows/test-action-linux.yml @@ -82,6 +82,7 @@ jobs: runs-on: ubuntu-latest outputs: matrix-include: ${{ steps.generate-matrix.outputs.matrix }} + build-reports: ${{ steps.generate-matrix.outputs.build-reports }} steps: - name: "Throttle Builds" @@ -272,53 +273,47 @@ jobs: - name: Upload Code Coverage Test Run Artifacts if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} + name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}${{ matrix.fips && '-fips ' || '' }} path: | artifacts/coverage/ - name: Upload JUnit XML Test Run Artifacts if: always() && steps.download-artifacts-from-vm.outcome == 'success' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }} + name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}${{ matrix.fips && '-fips ' || '' }} path: | artifacts/xml-unittests-output/ - name: Upload Test Run Log Artifacts if: always() && steps.download-artifacts-from-vm.outcome == 'success' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }} + name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}${{ matrix.fips && '-fips ' || '' }} path: | artifacts/logs report: name: Test Reports - if: always() && inputs.skip-code-coverage == false && needs.test.result != 'cancelled' && needs.test.result != 'skipped' + if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && inputs.skip-code-coverage == false && needs.test.result != 'cancelled' && needs.test.result != 'skipped' runs-on: ubuntu-latest needs: - test + - generate-matrix steps: - name: Checkout Source Code uses: actions/checkout@v4 - name: Download Code Coverage Test Run Artifacts - uses: actions/download-artifact@v3 - # This needs to be actions/download-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/download-artifact@v4 if: ${{ inputs.skip-code-coverage == false }} id: download-coverage-artifacts with: - name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} + pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + merge-multiple: true path: artifacts/coverage/ - name: Show Downloaded Test Run Artifacts @@ -356,9 +351,7 @@ jobs: - name: Upload Code Coverage DB if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: all-testrun-coverage-artifacts + name: all-testrun-coverage-artifacts-${{ inputs.distro-slug }}.${{ inputs.nox-session }} path: artifacts/coverage diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index 60cc01e4d7b..b7379964c2b 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -79,6 +79,7 @@ jobs: runs-on: ubuntu-latest outputs: matrix-include: ${{ steps.generate-matrix.outputs.matrix }} + build-reports: ${{ steps.generate-matrix.outputs.build-reports }} steps: - name: "Throttle Builds" @@ -300,53 +301,47 @@ jobs: - name: Upload Code Coverage Test Run Artifacts if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} + name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }} path: | artifacts/coverage/ - name: Upload JUnit XML Test Run Artifacts if: always() && steps.download-artifacts-from-vm.outcome == 'success' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }} + name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }} path: | artifacts/xml-unittests-output/ - name: Upload Test Run Log Artifacts if: always() && steps.download-artifacts-from-vm.outcome == 'success' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }} + name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }} path: | artifacts/logs report: name: Test Reports - if: always() && inputs.skip-code-coverage == false && needs.test.result != 'cancelled' && needs.test.result != 'skipped' + if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && inputs.skip-code-coverage == false && needs.test.result != 'cancelled' && needs.test.result != 'skipped' runs-on: ubuntu-latest needs: - test + - generate-matrix steps: - name: Checkout Source Code uses: actions/checkout@v4 - name: Download Code Coverage Test Run Artifacts - uses: actions/download-artifact@v3 - # This needs to be actions/download-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/download-artifact@v4 if: ${{ inputs.skip-code-coverage == false }} id: download-coverage-artifacts with: - name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} + pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + merge-multiple: true path: artifacts/coverage/ - name: Show Downloaded Test Run Artifacts @@ -389,9 +384,7 @@ jobs: - name: Upload Code Coverage DB if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: all-testrun-coverage-artifacts + name: all-testrun-coverage-artifacts-${{ inputs.distro-slug }}.${{ inputs.nox-session }} path: artifacts/coverage diff --git a/.github/workflows/test-action-windows.yml b/.github/workflows/test-action-windows.yml index 488bac9a136..0718b660e86 100644 --- a/.github/workflows/test-action-windows.yml +++ b/.github/workflows/test-action-windows.yml @@ -82,6 +82,7 @@ jobs: runs-on: ubuntu-latest outputs: matrix-include: ${{ steps.generate-matrix.outputs.matrix }} + build-reports: ${{ steps.generate-matrix.outputs.build-reports }} steps: - name: "Throttle Builds" @@ -272,54 +273,48 @@ jobs: - name: Upload Code Coverage Test Run Artifacts if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} + name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }} path: | artifacts/coverage/ - name: Upload JUnit XML Test Run Artifacts if: always() && steps.download-artifacts-from-vm.outcome == 'success' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }} + name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }} path: | artifacts/xml-unittests-output/ - name: Upload Test Run Log Artifacts if: always() && steps.download-artifacts-from-vm.outcome == 'success' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }} + name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }} path: | artifacts/logs report: name: Test Reports - if: always() && inputs.skip-code-coverage == false && needs.test.result != 'cancelled' && needs.test.result != 'skipped' + if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && inputs.skip-code-coverage == false && needs.test.result != 'cancelled' && needs.test.result != 'skipped' runs-on: ubuntu-latest needs: - test + - generate-matrix steps: - name: Checkout Source Code uses: actions/checkout@v4 - name: Download Code Coverage Test Run Artifacts - uses: actions/download-artifact@v3 - # This needs to be actions/download-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/download-artifact@v4 if: ${{ inputs.skip-code-coverage == false }} id: download-coverage-artifacts with: - name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }} + pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-* + merge-multiple: true path: artifacts/coverage/ - name: Show Downloaded Test Run Artifacts @@ -357,9 +352,7 @@ jobs: - name: Upload Code Coverage DB if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: all-testrun-coverage-artifacts + name: all-testrun-coverage-artifacts-${{ inputs.distro-slug }}.${{ inputs.nox-session }} path: artifacts/coverage diff --git a/.github/workflows/test-package-downloads-action.yml b/.github/workflows/test-package-downloads-action.yml index 6da3b87dcc0..7532813999a 100644 --- a/.github/workflows/test-package-downloads-action.yml +++ b/.github/workflows/test-package-downloads-action.yml @@ -55,8 +55,35 @@ env: jobs: + generate-matrix: + name: Generate Matrix + runs-on: ubuntu-latest + outputs: + matrix-include: ${{ steps.generate-matrix.outputs.matrix }} + steps: + + - name: "Throttle Builds" + shell: bash + run: | + t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t" + + - name: Checkout Source Code + uses: actions/checkout@v4 + + - name: Setup Python Tools Scripts + uses: ./.github/actions/setup-python-tools-scripts + with: + cache-prefix: ${{ inputs.cache-prefix }} + + - name: Generate Test Matrix + id: generate-matrix + run: | + tools ci pkg-downloads-matrix + linux: name: Linux + needs: + - generate-matrix runs-on: - self-hosted - linux @@ -68,115 +95,7 @@ jobs: strategy: fail-fast: false matrix: - include: - - distro-slug: amazonlinux-2 - arch: x86_64 - pkg-type: package - - distro-slug: amazonlinux-2-arm64 - arch: aarch64 - pkg-type: package - - distro-slug: amazonlinux-2-arm64 - arch: arm64 - pkg-type: package - - distro-slug: amazonlinux-2023 - arch: x86_64 - pkg-type: package - - distro-slug: amazonlinux-2023-arm64 - arch: aarch64 - pkg-type: package - - distro-slug: amazonlinux-2023-arm64 - arch: arm64 - pkg-type: package - - distro-slug: centos-7 - arch: x86_64 - pkg-type: package - - distro-slug: centos-7-arm64 - arch: aarch64 - pkg-type: package - - distro-slug: centos-7-arm64 - arch: arm64 - pkg-type: package - - distro-slug: debian-11 - arch: x86_64 - pkg-type: package - - distro-slug: debian-11-arm64 - arch: arm64 - pkg-type: package - - distro-slug: debian-12 - arch: x86_64 - pkg-type: package - - distro-slug: debian-12-arm64 - arch: arm64 - pkg-type: package - - distro-slug: fedora-39 - arch: x86_64 - pkg-type: package - - distro-slug: fedora-39-arm64 - arch: aarch64 - pkg-type: package - - distro-slug: fedora-39-arm64 - arch: arm64 - pkg-type: package - - distro-slug: photonos-4 - arch: x86_64 - pkg-type: package - - distro-slug: photonos-4-arm64 - arch: aarch64 - pkg-type: package - - distro-slug: photonos-4-arm64 - arch: arm64 - pkg-type: package - - distro-slug: photonos-5 - arch: x86_64 - pkg-type: package - - distro-slug: photonos-5-arm64 - arch: aarch64 - pkg-type: package - - distro-slug: photonos-5-arm64 - arch: arm64 - pkg-type: package - - distro-slug: rockylinux-8 - arch: x86_64 - pkg-type: package - - distro-slug: rockylinux-8-arm64 - arch: aarch64 - pkg-type: package - - distro-slug: rockylinux-8-arm64 - arch: arm64 - pkg-type: package - - distro-slug: rockylinux-9 - arch: x86_64 - pkg-type: package - - distro-slug: rockylinux-9-arm64 - arch: aarch64 - pkg-type: package - - distro-slug: rockylinux-9-arm64 - arch: arm64 - pkg-type: package - - distro-slug: ubuntu-20.04 - arch: x86_64 - pkg-type: package - - distro-slug: ubuntu-20.04-arm64 - arch: arm64 - pkg-type: package - - distro-slug: ubuntu-22.04 - arch: x86_64 - pkg-type: package - - distro-slug: ubuntu-22.04 - arch: x86_64 - pkg-type: onedir - - distro-slug: ubuntu-22.04-arm64 - arch: arm64 - pkg-type: package - - distro-slug: ubuntu-22.04-arm64 - arch: arm64 - pkg-type: onedir - - distro-slug: ubuntu-23.04 - arch: x86_64 - pkg-type: package - - distro-slug: ubuntu-23.04-arm64 - arch: arm64 - pkg-type: package + include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['linux'] }} steps: @@ -361,11 +280,9 @@ jobs: - name: Upload Test Run Artifacts if: always() && steps.download-artifacts-from-vm.outcome == 'success' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: pkg-testrun-artifacts-${{ matrix.distro-slug }}-${{ matrix.arch }} + name: pkg-testrun-artifacts-${{ matrix.distro-slug }}-${{ matrix.arch }}-${{ matrix.pkg-type }} path: | artifacts !artifacts/salt/* @@ -374,6 +291,8 @@ jobs: macos: name: MacOS + needs: + - generate-matrix runs-on: ${{ matrix.distro-slug == 'macos-13-arm64' && 'macos-13-xlarge' || matrix.distro-slug }} env: USE_S3_CACHE: 'false' @@ -382,19 +301,7 @@ jobs: strategy: fail-fast: false matrix: - include: - - distro-slug: macos-12 - arch: x86_64 - pkg-type: package - - distro-slug: macos-13 - arch: x86_64 - pkg-type: package - - distro-slug: macos-13-arm64 - arch: arm64 - pkg-type: package - - distro-slug: macos-13-arm64 - arch: arm64 - pkg-type: onedir + include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['macos'] }} steps: @@ -570,11 +477,9 @@ jobs: - name: Upload Test Run Artifacts if: always() - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: pkg-testrun-artifacts-${{ matrix.distro-slug }}-${{ matrix.arch }} + name: pkg-testrun-artifacts-${{ matrix.distro-slug }}-${{ matrix.arch }}-${{ matrix.pkg-type }} path: | artifacts !artifacts/salt/* @@ -583,6 +488,8 @@ jobs: windows: name: Windows + needs: + - generate-matrix env: USE_S3_CACHE: 'true' runs-on: @@ -594,16 +501,7 @@ jobs: strategy: fail-fast: false matrix: - include: - - distro-slug: windows-2022 - arch: amd64 - pkg-type: nsis - - distro-slug: windows-2022 - arch: amd64 - pkg-type: msi - - distro-slug: windows-2022 - arch: amd64 - pkg-type: onedir + include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['windows'] }} steps: - name: Checkout Source Code @@ -782,11 +680,9 @@ jobs: - name: Upload Test Run Artifacts if: always() && steps.download-artifacts-from-vm.outcome == 'success' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: pkg-testrun-artifacts-${{ matrix.distro-slug }}-${{ matrix.arch }} + name: pkg-testrun-artifacts-${{ matrix.distro-slug }}-${{ matrix.arch }}-${{ matrix.pkg-type }} path: | artifacts !artifacts/salt/* diff --git a/.github/workflows/test-packages-action-linux.yml b/.github/workflows/test-packages-action-linux.yml index 4032f4b581b..2197a662318 100644 --- a/.github/workflows/test-packages-action-linux.yml +++ b/.github/workflows/test-packages-action-linux.yml @@ -200,7 +200,7 @@ jobs: run: | tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install ${{ matrix.fips && '--fips ' || '' }}\ --nox-session=${{ inputs.nox-session }}-pkgs --rerun-failures ${{ inputs.distro-slug }} -- ${{ matrix.tests-chunk }} \ - ${{ matrix.version && format('--prev-version {0}', matrix.version) || ''}} + ${{ matrix.version && format('--prev-version={0}', matrix.version) || ''}} - name: Download Test Run Artifacts id: download-artifacts-from-vm @@ -219,11 +219,9 @@ jobs: - name: Upload Test Run Artifacts if: always() && steps.download-artifacts-from-vm.outcome == 'success' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ matrix.tests-chunk }} + name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.arch }}-${{ inputs.pkg-type }}-${{ matrix.tests-chunk }}-${{ matrix.version || 'no-version'}} path: | artifacts !artifacts/pkg/* @@ -248,11 +246,9 @@ jobs: - name: Download Test Run Artifacts id: download-test-run-artifacts - uses: actions/download-artifact@v3 - # This needs to be actions/download-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/download-artifact@v4 with: - name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ matrix.tests-chunk }} + name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.arch }}-${{ inputs.pkg-type }}-${{ matrix.tests-chunk }}-${{ matrix.version || 'no-version'}} path: artifacts - name: Show Test Run Artifacts diff --git a/.github/workflows/test-packages-action-macos.yml b/.github/workflows/test-packages-action-macos.yml index 611d84e22e7..686295cb17b 100644 --- a/.github/workflows/test-packages-action-macos.yml +++ b/.github/workflows/test-packages-action-macos.yml @@ -185,7 +185,7 @@ jobs: COVERAGE_CONTEXT: ${{ inputs.distro-slug }} run: | sudo -E nox --force-color -e ${{ inputs.nox-session }}-pkgs -- ${{ matrix.tests-chunk }} \ - ${{ matrix.version && format('--prev-version {0}', matrix.version) || ''}} + ${{ matrix.version && format('--prev-version={0}', matrix.version) || ''}} - name: Fix file ownership run: | @@ -202,11 +202,9 @@ jobs: - name: Upload Test Run Artifacts if: always() - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ matrix.tests-chunk }} + name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.arch }}-${{ inputs.pkg-type }}-${{ matrix.tests-chunk }}-${{ matrix.version || 'no-version'}} path: | artifacts !artifacts/pkg/* @@ -231,11 +229,9 @@ jobs: - name: Download Test Run Artifacts id: download-test-run-artifacts - uses: actions/download-artifact@v3 - # This needs to be actions/download-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/download-artifact@v4 with: - name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ matrix.tests-chunk }} + name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.arch }}-${{ inputs.pkg-type }}-${{ matrix.tests-chunk }}-${{ matrix.version || 'no-version'}} path: artifacts - name: Show Test Run Artifacts diff --git a/.github/workflows/test-packages-action-windows.yml b/.github/workflows/test-packages-action-windows.yml index bc04dbffc13..b8d2f21d5bd 100644 --- a/.github/workflows/test-packages-action-windows.yml +++ b/.github/workflows/test-packages-action-windows.yml @@ -199,7 +199,7 @@ jobs: run: | tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install ${{ matrix.fips && '--fips ' || '' }}\ --nox-session=${{ inputs.nox-session }}-pkgs --rerun-failures ${{ inputs.distro-slug }} -- ${{ matrix.tests-chunk }} \ - ${{ matrix.version && format('--prev-version {0}', matrix.version) || ''}} + ${{ matrix.version && format('--prev-version={0}', matrix.version) || ''}} - name: Download Test Run Artifacts id: download-artifacts-from-vm @@ -218,11 +218,9 @@ jobs: - name: Upload Test Run Artifacts if: always() && steps.download-artifacts-from-vm.outcome == 'success' - uses: actions/upload-artifact@v3 - # This needs to be actions/upload-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/upload-artifact@v4 with: - name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ matrix.tests-chunk }} + name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.arch }}-${{ inputs.pkg-type }}-${{ matrix.tests-chunk }}-${{ matrix.version || 'no-version'}} path: | artifacts !artifacts/pkg/* @@ -247,11 +245,9 @@ jobs: - name: Download Test Run Artifacts id: download-test-run-artifacts - uses: actions/download-artifact@v3 - # This needs to be actions/download-artifact@v3 because we upload multiple artifacts - # under the same name something that actions/upload-artifact@v4 does not do. + uses: actions/download-artifact@v4 with: - name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ matrix.tests-chunk }} + name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.arch }}-${{ inputs.pkg-type }}-${{ matrix.tests-chunk }}-${{ matrix.version || 'no-version'}} path: artifacts - name: Show Test Run Artifacts diff --git a/noxfile.py b/noxfile.py index f190e566938..fa7f95fe866 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1865,32 +1865,25 @@ def ci_test_onedir_pkgs(session): ] chunks = { - "install": [ - "tests/pytests/pkg/", - ], + "install": [], "upgrade": [ "--upgrade", "--no-uninstall", - "tests/pytests/pkg/upgrade/", ], "upgrade-classic": [ "--upgrade", "--no-uninstall", - "tests/pytests/pkg/upgrade/", ], "downgrade": [ "--downgrade", "--no-uninstall", - "tests/pytests/pkg/downgrade/", ], "downgrade-classic": [ "--downgrade", "--no-uninstall", - "tests/pytests/pkg/downgrade/", ], "download-pkgs": [ "--download-pkgs", - "tests/pytests/pkg/download/", ], } @@ -1931,6 +1924,17 @@ def ci_test_onedir_pkgs(session): ] + session.posargs ) + append_tests_path = True + test_paths = ( + "tests/pytests/pkg/", + str(REPO_ROOT / "tests" / "pytests" / "pkg"), + ) + for arg in session.posargs: + if arg.startswith(test_paths): + append_tests_path = False + break + if append_tests_path: + pytest_args.append("tests/pytests/pkg/") try: _pytest(session, coverage=False, cmd_args=pytest_args, env=env) except CommandFailed: @@ -1954,6 +1958,8 @@ def ci_test_onedir_pkgs(session): ] + session.posargs ) + if append_tests_path: + pytest_args.append("tests/pytests/pkg/") _pytest( session, coverage=False, @@ -1978,9 +1984,14 @@ def ci_test_onedir_pkgs(session): pytest_args.append("--use-prev-version") if chunk in ("upgrade-classic", "downgrade-classic"): pytest_args.append("--classic") + if append_tests_path: + pytest_args.append("tests/pytests/pkg/") try: _pytest(session, coverage=False, cmd_args=pytest_args, env=env) except CommandFailed: + if os.environ.get("RERUN_FAILURES", "0") == "0": + # Don't rerun on failures + return cmd_args = chunks["install"] pytest_args = ( common_pytest_args[:] @@ -1997,6 +2008,8 @@ def ci_test_onedir_pkgs(session): pytest_args.append("--use-prev-version") if chunk in ("upgrade-classic", "downgrade-classic"): pytest_args.append("--classic") + if append_tests_path: + pytest_args.append("tests/pytests/pkg/") _pytest( session, coverage=False, diff --git a/tests/pytests/pkg/conftest.py b/tests/pytests/pkg/conftest.py index 20d71f14228..5bcd544c119 100644 --- a/tests/pytests/pkg/conftest.py +++ b/tests/pytests/pkg/conftest.py @@ -11,6 +11,7 @@ from pytestskipmarkers.utils import platform from saltfactories.utils import random_string import salt.config +import salt.utils.files from tests.conftest import CODE_DIR from tests.support.pkg import ApiRequest, SaltMaster, SaltMasterWindows, SaltPkgInstall @@ -113,13 +114,66 @@ def pytest_addoption(parser): ) +@pytest.hookimpl(hookwrapper=True, trylast=True) +def pytest_collection_modifyitems(config, items): + """ + called after collection has been performed, may filter or re-order + the items in-place. + + :param _pytest.main.Session session: the pytest session object + :param _pytest.config.Config config: pytest config object + :param List[_pytest.nodes.Item] items: list of item objects + """ + # Let PyTest or other plugins handle the initial collection + yield + selected = [] + deselected = [] + pkg_tests_path = pathlib.Path(__file__).parent + + if config.getoption("--upgrade"): + for item in items: + if str(item.fspath).startswith(str(pkg_tests_path / "upgrade")): + selected.append(item) + else: + deselected.append(item) + elif config.getoption("--downgrade"): + for item in items: + if str(item.fspath).startswith(str(pkg_tests_path / "downgrade")): + selected.append(item) + else: + deselected.append(item) + elif config.getoption("--download-pkgs"): + for item in items: + if str(item.fspath).startswith(str(pkg_tests_path / "download")): + selected.append(item) + else: + deselected.append(item) + else: + exclude_paths = ( + str(pkg_tests_path / "upgrade"), + str(pkg_tests_path / "downgrade"), + str(pkg_tests_path / "download"), + ) + for item in items: + if str(item.fspath).startswith(exclude_paths): + deselected.append(item) + else: + selected.append(item) + + if deselected: + # Selection changed + items[:] = selected + config.hook.pytest_deselected(items=deselected) + + @pytest.hookimpl(tryfirst=True) def pytest_runtest_setup(item): """ Fixtures injection based on markers or test skips based on CLI arguments """ + pkg_tests_path = pathlib.Path(__file__).parent if ( - str(item.fspath).startswith(str(pathlib.Path(__file__).parent / "download")) + str(item.fspath).startswith(str(pkg_tests_path / "download")) and item.config.getoption("--download-pkgs") is False ): raise pytest.skip.Exception( @@ -128,6 +182,17 @@ def pytest_runtest_setup(item): _use_item_location=True, ) + for key in ("upgrade", "downgrade"): + if ( + str(item.fspath).startswith(str(pkg_tests_path / key)) + and item.config.getoption(f"--{key}") is False + ): + raise pytest.skip.Exception( + f"The package {key} tests are disabled. Pass '--{key}' to pytest " + "to enable them.", + _use_item_location=True, + ) + @pytest.fixture(scope="session") def salt_factories_root_dir(request, tmp_path_factory): @@ -235,16 +300,20 @@ def salt_master(salt_factories, install_salt, pkg_tests_account): "fips_mode": FIPS_TESTRUN, "open_mode": True, } - test_user = False + salt_user_in_config_file = False master_config = install_salt.config_path / "master" - if master_config.exists(): + if master_config.exists() and master_config.stat().st_size: with salt.utils.files.fopen(master_config) as fp: data = yaml.safe_load(fp) - if data and "user" in data: - test_user = True + if data is None: + # File exists but is mostly commented out + data = {} + user_in_config_file = data.get("user") + if user_in_config_file and user_in_config_file == "salt": + salt_user_in_config_file = True # We are testing a different user, so we need to test the system # configs, or else permissions will not be correct. - config_overrides["user"] = data["user"] + config_overrides["user"] = user_in_config_file config_overrides["log_file"] = salt.config.DEFAULT_MASTER_OPTS.get( "log_file" ) @@ -270,8 +339,18 @@ def salt_master(salt_factories, install_salt, pkg_tests_account): pathlib.Path("/var", "cache", "salt", "master"), ] for _file in verify_files: - assert _file.owner() == "salt" - assert _file.group() == "salt" + if _file.owner() != "salt": + log.warning( + "The owner of '%s' is '%s' when it should be 'salt'", + _file, + _file.owner(), + ) + if _file.group() != "salt": + log.warning( + "The group of '%s' is '%s' when it should be 'salt'", + _file, + _file.group(), + ) master_script = False if platform.is_windows(): @@ -321,7 +400,7 @@ def salt_master(salt_factories, install_salt, pkg_tests_account): salt_pkg_install=install_salt, ) factory.after_terminate(pytest.helpers.remove_stale_master_key, factory) - if test_user: + if salt_user_in_config_file: # Salt factories calls salt.utils.verify.verify_env # which sets root perms on /etc/salt/pki/master since we are running # the test suite as root, but we want to run Salt master as salt @@ -397,13 +476,21 @@ def salt_minion(salt_factories, salt_master, install_salt): # which sets root perms on /srv/salt and /srv/pillar since we are running # the test suite as root, but we want to run Salt master as salt if not platform.is_windows() and not platform.is_darwin(): - state_tree = "/srv/salt" - pillar_tree = "/srv/pillar" - check_paths = [state_tree, pillar_tree, CODE_DIR / ".nox"] - for path in check_paths: - if os.path.exists(path) is False: - continue - subprocess.run(["chown", "-R", "salt:salt", str(path)], check=False) + import pwd + + try: + pwd.getpwnam("salt") + except KeyError: + # The salt user does not exist + pass + else: + state_tree = "/srv/salt" + pillar_tree = "/srv/pillar" + check_paths = [state_tree, pillar_tree, CODE_DIR / ".nox"] + for path in check_paths: + if os.path.exists(path) is False: + continue + subprocess.run(["chown", "-R", "salt:salt", str(path)], check=False) factory.after_terminate( pytest.helpers.remove_stale_minion_key, salt_master, factory.id diff --git a/tests/pytests/pkg/downgrade/test_salt_downgrade.py b/tests/pytests/pkg/downgrade/test_salt_downgrade.py index 43fe1cc30dd..adba7c51272 100644 --- a/tests/pytests/pkg/downgrade/test_salt_downgrade.py +++ b/tests/pytests/pkg/downgrade/test_salt_downgrade.py @@ -1,6 +1,5 @@ import packaging.version import psutil -import pytest from pytestskipmarkers.utils import platform @@ -8,9 +7,6 @@ def test_salt_downgrade(salt_call_cli, install_salt): """ Test an upgrade of Salt. """ - if not install_salt.downgrade: - pytest.skip("Not testing a downgrade, do not run") - is_downgrade_to_relenv = packaging.version.parse( install_salt.prev_version ) >= packaging.version.parse("3006.0") diff --git a/tests/pytests/pkg/integration/test_salt_user.py b/tests/pytests/pkg/integration/test_salt_user.py index 74bf458a00b..834fd399121 100644 --- a/tests/pytests/pkg/integration/test_salt_user.py +++ b/tests/pytests/pkg/integration/test_salt_user.py @@ -11,6 +11,13 @@ from saltfactories.utils.tempfiles import temp_directory pytestmark = [ pytest.mark.skip_on_windows, pytest.mark.skip_on_darwin, + pytest.mark.skipif( + True, + reason=( + "Package permissions are getting reworked in " + "https://github.com/saltstack/salt/pull/66218" + ), + ), ] diff --git a/tests/pytests/pkg/upgrade/test_salt_upgrade.py b/tests/pytests/pkg/upgrade/test_salt_upgrade.py index d376d581adb..fd883705c4a 100644 --- a/tests/pytests/pkg/upgrade/test_salt_upgrade.py +++ b/tests/pytests/pkg/upgrade/test_salt_upgrade.py @@ -1,25 +1,38 @@ +import logging + import packaging.version import psutil -import pytest from pytestskipmarkers.utils import platform +log = logging.getLogger(__name__) + + +def _get_running_salt_minion_pid(process_name): + # psutil process name only returning first part of the command '/opt/saltstack/' + # need to check all of command line for salt-minion + # ['/opt/saltstack/salt/bin/python3.10 /usr/bin/salt-minion MultiMinionProcessManager MinionProcessManager'] + # and psutil is only returning the salt-minion once + pids = [] + for proc in psutil.process_iter(): + if "salt" in proc.name(): + cmdl_strg = " ".join(str(element) for element in proc.cmdline()) + if process_name in cmdl_strg: + pids.append(proc.pid) + return pids + def test_salt_upgrade(salt_call_cli, install_salt): """ Test an upgrade of Salt. """ - if not install_salt.upgrade: - pytest.skip("Not testing an upgrade, do not run") - if install_salt.relenv: original_py_version = install_salt.package_python_version() # Verify previous install version is setup correctly and works - ret = salt_call_cli.run("test.version") + ret = salt_call_cli.run("--local", "test.version") assert ret.returncode == 0 - assert packaging.version.parse(ret.data) < packaging.version.parse( - install_salt.artifact_version - ) + installed_version = packaging.version.parse(ret.data) + assert installed_version < packaging.version.parse(install_salt.artifact_version) # Test pip install before an upgrade dep = "PyGithub==1.56.0" @@ -32,45 +45,38 @@ def test_salt_upgrade(salt_call_cli, install_salt): assert "Authentication information could" in use_lib.stderr # Verify there is a running minion by getting its PID - salt_name = "salt" - if platform.is_windows(): - process_name = "salt-minion.exe" + if installed_version < packaging.version.parse("3006.0"): + # This is using PyInstaller + process_name = "run minion" else: - process_name = "salt-minion" - - old_pid = [] - - # psutil process name only returning first part of the command '/opt/saltstack/' - # need to check all of command line for salt-minion - # ['/opt/saltstack/salt/bin/python3.10 /usr/bin/salt-minion MultiMinionProcessManager MinionProcessManager'] - # and psutil is only returning the salt-minion once - for proc in psutil.process_iter(): - if salt_name in proc.name(): - cmdl_strg = " ".join(str(element) for element in proc.cmdline()) - if process_name in cmdl_strg: - old_pid.append(proc.pid) - - assert old_pid + if platform.is_windows(): + process_name = "salt-minion.exe" + else: + process_name = "salt-minion" + old_pids = _get_running_salt_minion_pid(process_name) + assert old_pids # Upgrade Salt from previous version and test install_salt.install(upgrade=True) - ret = salt_call_cli.run("test.version") + ret = salt_call_cli.run("--local", "test.version") assert ret.returncode == 0 - assert packaging.version.parse(ret.data) == packaging.version.parse( - install_salt.artifact_version - ) + installed_version = packaging.version.parse(ret.data) + assert installed_version == packaging.version.parse(install_salt.artifact_version) # Verify there is a new running minion by getting its PID and comparing it # with the PID from before the upgrade - new_pid = [] - for proc in psutil.process_iter(): - if salt_name in proc.name(): - cmdl_strg = " ".join(str(element) for element in proc.cmdline()) - if process_name in cmdl_strg: - new_pid.append(proc.pid) + if installed_version < packaging.version.parse("3006.0"): + # This is using PyInstaller + process_name = "run minion" + else: + if platform.is_windows(): + process_name = "salt-minion.exe" + else: + process_name = "salt-minion" + new_pids = _get_running_salt_minion_pid(process_name) - assert new_pid - assert new_pid != old_pid + assert new_pids + assert new_pids != old_pids if install_salt.relenv: new_py_version = install_salt.package_python_version() diff --git a/tests/support/pkg.py b/tests/support/pkg.py index 1eff0807868..7873bf182eb 100644 --- a/tests/support/pkg.py +++ b/tests/support/pkg.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Dict, List import attr import distro -import packaging +import packaging.version import psutil import pytest import requests @@ -22,6 +22,7 @@ from pytestshellutils.utils.processes import ( ProcessResult, _get_cmdline, terminate_process, + terminate_process_list, ) from pytestskipmarkers.utils import platform from saltfactories.bases import SystemdSaltDaemonImpl @@ -85,7 +86,7 @@ class SaltPkgInstall: @proc.default def _default_proc(self): - return Subprocess() + return Subprocess(timeout=240) @distro_id.default def _default_distro_id(self): @@ -105,7 +106,7 @@ class SaltPkgInstall: @distro_version.default def _default_distro_version(self): - if self.distro_name == "photon": + if self.distro_name in ("photon", "rocky"): return distro.version().split(".")[0] return distro.version().lower() @@ -514,7 +515,6 @@ class SaltPkgInstall: upgrade_cmd, "-y", *args, - _timeout=120, env=env, ) else: @@ -1025,6 +1025,17 @@ class SaltPkgInstall: if not self.no_uninstall: self.uninstall() + # Did we left anything running?! + procs = [] + for proc in psutil.process_iter(): + if "salt" in proc.name(): + cmdl_strg = " ".join(str(element) for element in _get_cmdline(proc)) + if "/opt/saltstack" in cmdl_strg: + procs.append(proc) + + if procs: + terminate_process_list(procs, kill=True, slow_stop=True) + class PkgSystemdSaltDaemonImpl(SystemdSaltDaemonImpl): # pylint: disable=access-member-before-definition diff --git a/tools/ci.py b/tools/ci.py index 61d9d71e7ad..e242fc54a86 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -20,6 +20,7 @@ from ptscripts import Context, command_group import tools.utils import tools.utils.gh +from tools.precommit.workflows import TEST_SALT_LISTING if sys.version_info < (3, 11): from typing_extensions import NotRequired, TypedDict @@ -658,10 +659,28 @@ def matrix( """ Generate the test matrix. """ + gh_event_path = os.environ.get("GITHUB_EVENT_PATH") or None + if gh_event_path is None: + ctx.warn("The 'GITHUB_EVENT_PATH' variable is not set.") + ctx.exit(1) + + if TYPE_CHECKING: + assert gh_event_path is not None + + gh_event = None + try: + gh_event = json.loads(open(gh_event_path, encoding="utf-8").read()) + except Exception as exc: + ctx.error(f"Could not load the GH Event payload from {gh_event_path!r}:\n", exc) + ctx.exit(1) + + if TYPE_CHECKING: + assert gh_event is not None + _matrix = [] _splits = { "functional": 4, - "integration": 6, + "integration": 7, "scenarios": 1, "unit": 4, } @@ -711,10 +730,19 @@ def matrix( ctx.info("Generated matrix:") ctx.print(_matrix, soft_wrap=True) + if ( + gh_event["repository"]["fork"] is True + and "macos" in distro_slug + and "arm64" in distro_slug + ): + ctx.warn("Forks don't have access to MacOS 13 Arm64. Clearning the matrix.") + _matrix.clear() + github_output = os.environ.get("GITHUB_OUTPUT") if github_output is not None: with open(github_output, "a", encoding="utf-8") as wfh: wfh.write(f"matrix={json.dumps(_matrix)}\n") + wfh.write(f"build-reports={json.dumps(len(_matrix) > 0)}\n") ctx.exit(0) @@ -747,9 +775,22 @@ def pkg_matrix( """ Generate the test matrix. """ + gh_event = None + gh_event_path = os.environ.get("GITHUB_EVENT_PATH") or None + if gh_event_path is None: + ctx.warn("The 'GITHUB_EVENT_PATH' variable is not set.") + else: + try: + gh_event = json.loads(open(gh_event_path, encoding="utf-8").read()) + except Exception as exc: + ctx.error( + f"Could not load the GH Event payload from {gh_event_path!r}:\n", exc + ) + github_output = os.environ.get("GITHUB_OUTPUT") if github_output is None: ctx.warn("The 'GITHUB_OUTPUT' variable is not set.") + if TYPE_CHECKING: assert testing_releases @@ -821,12 +862,22 @@ def pkg_matrix( paginator = s3.get_paginator("list_objects_v2") _matrix = [ { - "test-chunk": "install", + "tests-chunk": "install", "version": None, } ] for version, backend in adjusted_versions: + if ( + distro_slug.startswith(("macos-", "debian-", "ubuntu-")) + or version.major < 3006 + ): + # XXX: Temporarily skip problematic tests + ctx.warn( + f"Temporary skip builds on {distro_slug} for version {version} with backend {backend}" + ) + continue + prefix = prefixes[backend] # TODO: Remove this after 3009.0 if backend == "relenv" and version >= tools.utils.Version("3006.5"): @@ -843,9 +894,11 @@ def pkg_matrix( # key_filter = f"Contents[?contains(Key, '{version}')] | [?ends_with(Key, '.msi')]" continue elif pkg_type == "NSIS": - key_filter = ( - f"Contents[?contains(Key, '{version}')] | [?ends_with(Key, '.exe')]" - ) + # XXX: Temporarily skip problematic tests + # key_filter = ( + # f"Contents[?contains(Key, '{version}')] | [?ends_with(Key, '.exe')]" + # ) + continue objects = list(page_iterator.search(key_filter)) # Testing using `any` because sometimes the paginator returns `[None]` if any(objects): @@ -857,7 +910,7 @@ def pkg_matrix( session += "-classic" _matrix.append( { - "test-chunk": session, + "tests-chunk": session, "version": str(version), } ) @@ -874,6 +927,176 @@ def pkg_matrix( ctx.info("Generated matrix:") ctx.print(_matrix, soft_wrap=True) + if ( + gh_event is not None + and gh_event["repository"]["fork"] is True + and "macos" in distro_slug + and "arm64" in distro_slug + ): + ctx.warn("Forks don't have access to MacOS 13 Arm64. Clearning the matrix.") + _matrix.clear() + + if github_output is not None: + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"matrix={json.dumps(_matrix)}\n") + wfh.write(f"build-reports={json.dumps(len(_matrix) > 0)}\n") + ctx.exit(0) + + +@ci.command(name="deps-matrix") +def get_ci_deps_matrix(ctx: Context): + gh_event_path = os.environ.get("GITHUB_EVENT_PATH") or None + if gh_event_path is None: + ctx.warn("The 'GITHUB_EVENT_PATH' variable is not set.") + ctx.exit(1) + + if TYPE_CHECKING: + assert gh_event_path is not None + + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output is None: + ctx.warn("The 'GITHUB_OUTPUT' variable is not set.") + ctx.exit(1) + + if TYPE_CHECKING: + assert github_output is not None + + gh_event = None + try: + gh_event = json.loads(open(gh_event_path, encoding="utf-8").read()) + except Exception as exc: + ctx.error(f"Could not load the GH Event payload from {gh_event_path!r}:\n", exc) + ctx.exit(1) + + if TYPE_CHECKING: + assert gh_event is not None + + _matrix = { + "linux": [ + {"distro-slug": "centos-7", "arch": "x86_64"}, + {"distro-slug": "centos-7-arm64", "arch": "arm64"}, + ], + "macos": [ + {"distro-slug": "macos-12", "arch": "x86_64"}, + ], + "windows": [ + {"distro-slug": "windows-2022", "arch": "amd64"}, + ], + } + if gh_event["repository"]["fork"] is not True: + _matrix["macos"].append( + { + "distro-slug": "macos-13-arm64", + "arch": "arm64", + } + ) + + ctx.info("Generated matrix:") + ctx.print(_matrix, soft_wrap=True) + + if github_output is not None: + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"matrix={json.dumps(_matrix)}\n") + ctx.exit(0) + + +@ci.command(name="pkg-downloads-matrix") +def get_pkg_downloads_matrix(ctx: Context): + gh_event_path = os.environ.get("GITHUB_EVENT_PATH") or None + if gh_event_path is None: + ctx.warn("The 'GITHUB_EVENT_PATH' variable is not set.") + ctx.exit(1) + + if TYPE_CHECKING: + assert gh_event_path is not None + + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output is None: + ctx.warn("The 'GITHUB_OUTPUT' variable is not set.") + ctx.exit(1) + + if TYPE_CHECKING: + assert github_output is not None + + gh_event = None + try: + gh_event = json.loads(open(gh_event_path, encoding="utf-8").read()) + except Exception as exc: + ctx.error(f"Could not load the GH Event payload from {gh_event_path!r}:\n", exc) + ctx.exit(1) + + if TYPE_CHECKING: + assert gh_event is not None + + _matrix: dict[str, list[dict[str, str]]] = { + "linux": [], + "macos": [], + "windows": [], + } + + rpm_slugs = ( + "rockylinux", + "amazonlinux", + "centos", + "fedora", + "photon", + ) + linux_skip_pkg_download_tests = ( + "archlinux-lts", + "opensuse-15", + "windows", + ) + for slug in sorted(tools.utils.get_golden_images()): + if slug.startswith(linux_skip_pkg_download_tests): + continue + if "arm64" in slug: + arch = "arm64" + else: + arch = "x86_64" + if slug.startswith(rpm_slugs) and arch == "arm64": + # While we maintain backwards compatible urls + _matrix["linux"].append( + {"distro-slug": slug, "arch": "aarch64", "pkg-type": "package"} + ) + _matrix["linux"].append( + {"distro-slug": slug, "arch": arch, "pkg-type": "package"} + ) + if slug.startswith("ubuntu-22"): + _matrix["linux"].append( + {"distro-slug": slug, "arch": arch, "pkg-type": "onedir"} + ) + for mac in TEST_SALT_LISTING["macos"]: + if gh_event["repository"]["fork"] is True and mac.arch == "arm64": + continue + _matrix["macos"].append( + {"distro-slug": mac.slug, "arch": mac.arch, "pkg-type": "package"} + ) + + if gh_event["repository"]["fork"] is True: + macos_idx = 0 # macos-12 + else: + macos_idx = 1 # macos-13 + _matrix["macos"].append( + { + "distro-slug": TEST_SALT_LISTING["macos"][macos_idx].slug, + "arch": TEST_SALT_LISTING["macos"][macos_idx].arch, + "pkg-type": "onedir", + } + ) + + for win in TEST_SALT_LISTING["windows"][-1:]: + for pkg_type in ("nsis", "msi", "onedir"): + _matrix["windows"].append( + { + "distro-slug": win.slug, + "arch": win.arch, + "pkg-type": pkg_type, + } + ) + + ctx.info("Generated matrix:") + ctx.print(_matrix, soft_wrap=True) + if github_output is not None: with open(github_output, "a", encoding="utf-8") as wfh: wfh.write(f"matrix={json.dumps(_matrix)}\n") @@ -1168,8 +1391,6 @@ def get_testing_releases( """ Get a list of releases to use for the upgrade and downgrade tests. """ - # We aren't testing upgrades from anything before 3006.0 except the latest 3005.x - threshold_major = 3005 parsed_salt_version = tools.utils.Version(salt_version) # We want the latest 4 major versions, removing the oldest if this version is a new major num_major_versions = 4 @@ -1177,7 +1398,15 @@ def get_testing_releases( num_major_versions = 3 majors = sorted( list( - {version.major for version in releases if version.major >= threshold_major} + { + version.major + for version in releases + # We aren't testing upgrades from anything before + # 3006.0 except the latest 3005.x + if version.major >= 3005 + # We don't want to test 3007.? on the 3006.x branch + and version.major <= parsed_salt_version.major + } ) )[-num_major_versions:] testing_releases = [] diff --git a/tools/precommit/workflows.py b/tools/precommit/workflows.py index 4228253dd51..8642a74285f 100644 --- a/tools/precommit/workflows.py +++ b/tools/precommit/workflows.py @@ -7,31 +7,112 @@ from __future__ import annotations import logging import shutil -import sys from typing import TYPE_CHECKING, cast from jinja2 import Environment, FileSystemLoader, StrictUndefined from ptscripts import Context, command_group import tools.utils -from tools.utils import Linux, MacOS, Windows - -if sys.version_info < (3, 11): - from typing_extensions import TypedDict -else: - from typing import TypedDict # pylint: disable=no-name-in-module +from tools.utils import Linux, MacOS, PlatformDefinitions, Windows log = logging.getLogger(__name__) WORKFLOWS = tools.utils.REPO_ROOT / ".github" / "workflows" TEMPLATES = WORKFLOWS / "templates" - -class PlatformDefinitions(TypedDict): - linux: list[Linux] - macos: list[MacOS] - windows: list[Windows] - +TEST_SALT_LISTING = PlatformDefinitions( + { + "linux": [ + Linux(slug="rockylinux-8", display_name="Rocky Linux 8", arch="x86_64"), + Linux( + slug="rockylinux-8-arm64", + display_name="Rocky Linux 8 Arm64", + arch="arm64", + ), + Linux(slug="rockylinux-9", display_name="Rocky Linux 9", arch="x86_64"), + Linux( + slug="rockylinux-9-arm64", + display_name="Rocky Linux 9 Arm64", + arch="arm64", + ), + Linux(slug="amazonlinux-2", display_name="Amazon Linux 2", arch="x86_64"), + Linux( + slug="amazonlinux-2-arm64", + display_name="Amazon Linux 2 Arm64", + arch="arm64", + ), + Linux( + slug="amazonlinux-2023", + display_name="Amazon Linux 2023", + arch="x86_64", + ), + Linux( + slug="amazonlinux-2023-arm64", + display_name="Amazon Linux 2023 Arm64", + arch="arm64", + ), + Linux(slug="archlinux-lts", display_name="Arch Linux LTS", arch="x86_64"), + Linux(slug="centos-7", display_name="CentOS 7", arch="x86_64"), + Linux(slug="debian-11", display_name="Debian 11", arch="x86_64"), + Linux(slug="debian-11-arm64", display_name="Debian 11 Arm64", arch="arm64"), + Linux(slug="debian-12", display_name="Debian 12", arch="x86_64"), + Linux(slug="debian-12-arm64", display_name="Debian 12 Arm64", arch="arm64"), + Linux(slug="fedora-39", display_name="Fedora 39", arch="x86_64"), + Linux(slug="opensuse-15", display_name="Opensuse 15", arch="x86_64"), + Linux( + slug="photonos-4", + display_name="Photon OS 4", + arch="x86_64", + fips=True, + ), + Linux( + slug="photonos-4-arm64", + display_name="Photon OS 4 Arm64", + arch="arm64", + fips=True, + ), + Linux( + slug="photonos-5", + display_name="Photon OS 5", + arch="x86_64", + fips=True, + ), + Linux( + slug="photonos-5-arm64", + display_name="Photon OS 5 Arm64", + arch="arm64", + fips=True, + ), + Linux(slug="ubuntu-20.04", display_name="Ubuntu 20.04", arch="x86_64"), + Linux( + slug="ubuntu-20.04-arm64", + display_name="Ubuntu 20.04 Arm64", + arch="arm64", + ), + Linux(slug="ubuntu-22.04", display_name="Ubuntu 22.04", arch="x86_64"), + Linux( + slug="ubuntu-22.04-arm64", + display_name="Ubuntu 22.04 Arm64", + arch="arm64", + ), + ], + "macos": [ + MacOS(slug="macos-12", display_name="macOS 12", arch="x86_64"), + MacOS(slug="macos-13", display_name="macOS 13", arch="x86_64"), + MacOS( + slug="macos-13-arm64", + display_name="macOS 13 Arm64", + arch="arm64", + runner="macos-13-xlarge", + ), + ], + "windows": [ + Windows(slug="windows-2016", display_name="Windows 2016", arch="amd64"), + Windows(slug="windows-2019", display_name="Windows 2019", arch="amd64"), + Windows(slug="windows-2022", display_name="Windows 2022", arch="amd64"), + ], + } +) # Define the command group cgroup = command_group( @@ -97,114 +178,7 @@ def generate_workflows(ctx: Context): "test-pkg-downloads": True, }, }, - "Test Package Downloads": { - "template": "test-package-downloads-action.yml", - }, - "Build CI Deps": { - "template": "build-deps-ci-action.yml", - }, } - test_salt_listing = PlatformDefinitions( - { - "linux": [ - Linux(slug="rockylinux-8", display_name="Rocky Linux 8", arch="x86_64"), - Linux( - slug="rockylinux-8-arm64", - display_name="Rocky Linux 8 Arm64", - arch="arm64", - ), - Linux(slug="rockylinux-9", display_name="Rocky Linux 9", arch="x86_64"), - Linux( - slug="rockylinux-9-arm64", - display_name="Rocky Linux 9 Arm64", - arch="arm64", - ), - Linux( - slug="amazonlinux-2", display_name="Amazon Linux 2", arch="x86_64" - ), - Linux( - slug="amazonlinux-2-arm64", - display_name="Amazon Linux 2 Arm64", - arch="arm64", - ), - Linux( - slug="amazonlinux-2023", - display_name="Amazon Linux 2023", - arch="x86_64", - ), - Linux( - slug="amazonlinux-2023-arm64", - display_name="Amazon Linux 2023 Arm64", - arch="arm64", - ), - Linux( - slug="archlinux-lts", display_name="Arch Linux LTS", arch="x86_64" - ), - Linux(slug="centos-7", display_name="CentOS 7", arch="x86_64"), - Linux(slug="debian-11", display_name="Debian 11", arch="x86_64"), - Linux( - slug="debian-11-arm64", display_name="Debian 11 Arm64", arch="arm64" - ), - Linux(slug="debian-12", display_name="Debian 12", arch="x86_64"), - Linux( - slug="debian-12-arm64", display_name="Debian 12 Arm64", arch="arm64" - ), - Linux(slug="fedora-39", display_name="Fedora 39", arch="x86_64"), - Linux(slug="opensuse-15", display_name="Opensuse 15", arch="x86_64"), - Linux( - slug="photonos-4", - display_name="Photon OS 4", - arch="x86_64", - fips=True, - ), - Linux( - slug="photonos-4-arm64", - display_name="Photon OS 4 Arm64", - arch="arm64", - fips=True, - ), - Linux( - slug="photonos-5", - display_name="Photon OS 5", - arch="x86_64", - fips=True, - ), - Linux( - slug="photonos-5-arm64", - display_name="Photon OS 5 Arm64", - arch="arm64", - fips=True, - ), - Linux(slug="ubuntu-20.04", display_name="Ubuntu 20.04", arch="x86_64"), - Linux( - slug="ubuntu-20.04-arm64", - display_name="Ubuntu 20.04 Arm64", - arch="arm64", - ), - Linux(slug="ubuntu-22.04", display_name="Ubuntu 22.04", arch="x86_64"), - Linux( - slug="ubuntu-22.04-arm64", - display_name="Ubuntu 22.04 Arm64", - arch="arm64", - ), - ], - "macos": [ - MacOS(slug="macos-12", display_name="macOS 12", arch="x86_64"), - MacOS(slug="macos-13", display_name="macOS 13", arch="x86_64"), - MacOS( - slug="macos-13-arm64", - display_name="macOS 13 Arm64", - arch="arm64", - runner="macos-13-xlarge", - ), - ], - "windows": [ - Windows(slug="windows-2016", display_name="Windows 2016", arch="amd64"), - Windows(slug="windows-2019", display_name="Windows 2019", arch="amd64"), - Windows(slug="windows-2022", display_name="Windows 2022", arch="amd64"), - ], - } - ) test_salt_pkg_listing = PlatformDefinitions( { @@ -391,105 +365,6 @@ def generate_workflows(ctx: Context): } ) - build_ci_deps_listing = { - "linux": [ - ("x86_64", "centos-7"), - ("arm64", "centos-7-arm64"), - ], - "macos": [ - ("x86_64", "macos-12"), - ("arm64", "macos-13-arm64"), - ], - "windows": [ - ("amd64", "windows-2022"), - ], - } - test_salt_pkg_downloads_listing = PlatformDefinitions( - { - "linux": [], - "macos": [], - "windows": [], - } - ) - rpm_slugs = ( - "rockylinux", - "amazonlinux", - "centos", - "fedora", - "photon", - ) - linux_skip_pkg_download_tests = ( - "archlinux-lts", - "opensuse-15", - "windows", - ) - for slug in sorted(tools.utils.get_golden_images()): - if slug.startswith(linux_skip_pkg_download_tests): - continue - if "arm64" in slug: - arch = "arm64" - else: - arch = "x86_64" - if slug.startswith(rpm_slugs) and arch == "arm64": - # While we maintain backwards compatible urls - test_salt_pkg_downloads_listing["linux"].append( - Linux( - slug=slug, - arch="aarch64", - pkg_type="package", - ) - ) - test_salt_pkg_downloads_listing["linux"].append( - Linux( - slug=slug, - arch=arch, - pkg_type="package", - ) - ) - if slug.startswith("ubuntu-22"): - test_salt_pkg_downloads_listing["linux"].append( - Linux( - slug=slug, - arch=arch, - pkg_type="onedir", - ) - ) - for mac in test_salt_listing["macos"]: - test_salt_pkg_downloads_listing["macos"].append( - MacOS( - slug=mac.slug, - arch=mac.arch, - display_name=mac.display_name, - pkg_type="package", - runner=mac.runner, - ) - ) - for mac in test_salt_listing["macos"][-1:]: - test_salt_pkg_downloads_listing["macos"].append( - MacOS( - slug=mac.slug, - arch=mac.arch, - display_name=mac.display_name, - pkg_type="onedir", - runner=mac.runner, - ) - ) - for win in test_salt_listing["windows"][-1:]: - for pkg_type in ("nsis", "msi", "onedir"): - test_salt_pkg_downloads_listing["windows"].append( - Windows( - slug=win.slug, - arch=win.arch, - display_name=win.display_name, - pkg_type=pkg_type, - ) - ) - - test_salt_pkg_downloads_needs_slugs = {"build-ci-deps"} - # for platform in test_salt_pkg_downloads_listing: - # for _, arch, _ in test_salt_pkg_downloads_listing[platform]: - # test_salt_pkg_downloads_needs_slugs.add("build-ci-deps") - build_rpms_listing = [] rpm_os_versions: dict[str, list[str]] = { "amazon": [], @@ -563,13 +438,8 @@ def generate_workflows(ctx: Context): "test_repo_needs": NeedsTracker(), "prepare_workflow_needs": NeedsTracker(), "build_repo_needs": NeedsTracker(), - "test_salt_listing": test_salt_listing, + "test_salt_listing": TEST_SALT_LISTING, "test_salt_pkg_listing": test_salt_pkg_listing, - "build_ci_deps_listing": build_ci_deps_listing, - "test_salt_pkg_downloads_listing": test_salt_pkg_downloads_listing, - "test_salt_pkg_downloads_needs_slugs": sorted( - test_salt_pkg_downloads_needs_slugs - ), "build_rpms_listing": build_rpms_listing, "build_debs_listing": build_debs_listing, } diff --git a/tools/utils/__init__.py b/tools/utils/__init__.py index 0c4a71f67b5..6f79b4c4493 100644 --- a/tools/utils/__init__.py +++ b/tools/utils/__init__.py @@ -24,6 +24,11 @@ from rich.progress import ( TransferSpeedColumn, ) +if sys.version_info < (3, 11): + from typing_extensions import TypedDict +else: + from typing import TypedDict # pylint: disable=no-name-in-module + REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent GPG_KEY_FILENAME = "SALT-PROJECT-GPG-PUBKEY-2023" SPB_ENVIRONMENT = os.environ.get("SPB_ENVIRONMENT") or "test" @@ -45,10 +50,21 @@ class ExitCode(IntEnum): class OS: platform: str = attr.ib() slug: str = attr.ib() + arch: str = attr.ib() display_name: str = attr.ib(default=None) - arch: str = attr.ib(default=None) pkg_type: str = attr.ib(default=None) + @arch.default + def _default_arch(self): + return self._get_default_arch() + + def _get_default_arch(self): + if "aarch64" in self.slug: + return "arm64" + if "arm64" in self.slug: + return "arm64" + return "x86_64" + @attr.s(frozen=True, slots=True) class Linux(OS): @@ -70,6 +86,15 @@ class MacOS(OS): class Windows(OS): platform: str = attr.ib(default="windows") + def _get_default_arch(self): + return "amd64" + + +class PlatformDefinitions(TypedDict): + linux: list[Linux] + macos: list[MacOS] + windows: list[Windows] + def create_progress_bar(file_progress: bool = False, **kwargs): if file_progress: diff --git a/tools/vm.py b/tools/vm.py index b320cd10e36..d9e9c1e6e5b 100644 --- a/tools/vm.py +++ b/tools/vm.py @@ -1373,10 +1373,11 @@ class VM: for drive in ("c:", "C:"): rsync_remote_path = rsync_remote_path.replace(drive, "/cygdrive/c") destination = f"{self.name}:{rsync_remote_path}" - description = "Rsync local checkout to VM..." if download: + description = "Rsync VM to local checkout..." self.rsync(f"{destination}/*", source, description, rsync_flags) else: + description = "Rsync local checkout to VM..." self.rsync(source, destination, description, rsync_flags) if self.is_windows: # rsync sets very strict file permissions and disables inheritance