From 2e4314b30faa56c6e2d14400936a75f9a4fe1fac Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 17 Jan 2023 04:50:55 +0000 Subject: [PATCH] Start building onedir packages Signed-off-by: Pedro Algarvio --- .coveragerc | 1 + .github/actions/build-onedir-bare/action.yml | 36 ++ .github/actions/build-onedir-pkg/action.yml | 75 ++++ .github/actions/setup-relenv/action.yml | 46 +++ .github/workflows/build-bare-action.yml | 124 ++++++ .github/workflows/ci.yml | 373 +++++++++++++++++-- .github/workflows/dependencies-action.yml | 109 ++++++ .github/workflows/docs-action.yml | 4 +- .github/workflows/lint-action.yml | 4 +- .github/workflows/pre-commit-action.yml | 2 +- .github/workflows/test-action-macos.yml | 71 +++- .github/workflows/test-action.yml | 63 +++- noxfile.py | 66 +++- tools/vm.py | 10 +- 14 files changed, 917 insertions(+), 67 deletions(-) create mode 100644 .github/actions/build-onedir-bare/action.yml create mode 100644 .github/actions/build-onedir-pkg/action.yml create mode 100644 .github/actions/setup-relenv/action.yml create mode 100644 .github/workflows/build-bare-action.yml create mode 100644 .github/workflows/dependencies-action.yml diff --git a/.coveragerc b/.coveragerc index cb213274ae9..2579c96f9cd 100644 --- a/.coveragerc +++ b/.coveragerc @@ -32,6 +32,7 @@ ignore_errors = True [paths] salt = salt/ + artifacts/salt **/testing/salt/ **\testing\salt tests = diff --git a/.github/actions/build-onedir-bare/action.yml b/.github/actions/build-onedir-bare/action.yml new file mode 100644 index 00000000000..be262f9362d --- /dev/null +++ b/.github/actions/build-onedir-bare/action.yml @@ -0,0 +1,36 @@ +--- +name: build-onedir-bare +description: Build Bare Onedir Package +inputs: + platform: + required: true + type: string + description: The platform to build + arch: + required: true + type: string + description: The platform arch to build + package-name: + required: false + type: string + description: The onedir package name to create + default: salt + +runs: + using: composite + + steps: + + - name: Cache Bare Onedir Package Directory + id: onedir-pkg-cache + uses: actions/cache@v3 + with: + path: artifacts/${{ inputs.package-name }} + key: relenv|${{ env.RELENV_VERSION }}|bare|${{ inputs.platform }}|${{ inputs.arch }}|${{ inputs.package-name }}|${{ hashFiles('.relenv/**/*.xz') }} + + - name: Create Onedir Directory + shell: bash + if: steps.onedir-pkg-cache.outputs.cache-hit != 'true' + run: | + python3 -c "import os; os.makedirs('artifacts', exist_ok=True)" + python3 -m relenv create --arch=${{ inputs.arch }} artifacts/${{ inputs.package-name }} diff --git a/.github/actions/build-onedir-pkg/action.yml b/.github/actions/build-onedir-pkg/action.yml new file mode 100644 index 00000000000..a026c9073f3 --- /dev/null +++ b/.github/actions/build-onedir-pkg/action.yml @@ -0,0 +1,75 @@ +--- +name: build-linux-onedir +description: Build Linux Onedir Package +inputs: + platform: + required: true + type: string + description: The platform to build + arch: + required: true + type: string + description: The platform arch to build + package-name: + required: false + type: string + description: The onedir package name to create + default: salt + +runs: + using: composite + + steps: + + - name: Download Cached Bare Onedir Package Directory + id: onedir-bare-cache + uses: actions/cache@v3 + with: + path: artifacts/${{ inputs.package-name }} + key: relenv|${{ env.RELENV_VERSION }}|bare|${{ inputs.platform }}|${{ inputs.arch }}|${{ inputs.package-name }}|${{ hashFiles('.relenv/**/*.xz') }} + + - name: Cache Pip Wheels + uses: actions/cache@v3 + with: + path: .pip-download-cache + key: pip-download-cache|${{ env.RELENV_VERSION }}|salt|${{ inputs.platform }}|${{ inputs.arch }}|${{ inputs.package-name }}|${{ hashFiles('.relenv/**/*.xz', 'requirements/static/pkg/*/*.txt') }} + + - name: Install Salt Onedir Package Dependencies + env: + PIP_DOWNLOAD_CACHE: .pip-download-cache + PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ + PIP_EXTRA_INDEX_URL: https://pypi.org/simple + shell: bash + run: | + python3 -c "import os; os.makedirs('.downloaded-dependencies', exist_ok=True)" + if [ "${{ inputs.platform }}" != "windows" ]; then + artifacts/${{ inputs.package-name }}/bin/python3 -m pip install -r requirements/static/pkg/py3.10/${{ inputs.platform }}.txt + else + artifacts/${{ inputs.package-name }}/Scripts/python -m pip install -r requirements/static/pkg/py3.10/${{ inputs.platform }}.txt + fi + + - name: Install Salt Into Onedir + env: + PIP_DOWNLOAD_CACHE: .pip-download-cache + PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ + PIP_EXTRA_INDEX_URL: https://pypi.org/simple + shell: bash + run: | + if [ "${{ inputs.platform }}" != "windows" ]; then + artifacts/${{ inputs.package-name }}/bin/python3 -m pip install . + else + artifacts/${{ inputs.package-name }}/Scripts/python -m pip install . + fi + + - name: Create Tarball + shell: bash + run: | + cd artifacts/ + tar -cJf ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz ${{ inputs.package-name }} + + - name: Upload Onedir Tarball as an Artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz + path: artifacts/${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz + retention-days: 7 diff --git a/.github/actions/setup-relenv/action.yml b/.github/actions/setup-relenv/action.yml new file mode 100644 index 00000000000..70dd341050a --- /dev/null +++ b/.github/actions/setup-relenv/action.yml @@ -0,0 +1,46 @@ +--- +name: setup-relenv +description: Setup Relenv +inputs: + platform: + required: true + type: string + description: The platform to build + arch: + required: true + type: string + description: The platform arch to build + +runs: + using: composite + + steps: + + - name: Install Relenv + shell: bash + run: | + python3 -m pip install relenv + + - name: Set environment variables + shell: bash + run: | + python3 -c "import os; os.makedirs('.relenv', exist_ok=True)" + echo RELENV_DATA="$(pwd)/.relenv" >> $GITHUB_ENV + echo RELENV_VERSION="$(python3 -m pip show relenv | grep Version | cut -d ' ' -f2)" >> $GITHUB_ENV + + - name: Cache Relenv Data Directory + uses: actions/cache@v3 + with: + path: ${{ env.RELENV_DATA }} + key: relenv|${{ env.RELENV_VERSION }}|${{ inputs.platform }}|${{ inputs.arch }} }} + + - name: Fetch Toolchain + if: ${{ inputs.platform == 'linux' }} + shell: bash + run: | + python3 -m relenv toolchain fetch --arch=${{ inputs.arch }} + + - name: Fetch Native Python Build + shell: bash + run: | + python3 -m relenv fetch --arch=${{ inputs.arch }} diff --git a/.github/workflows/build-bare-action.yml b/.github/workflows/build-bare-action.yml new file mode 100644 index 00000000000..3b70af0b0c6 --- /dev/null +++ b/.github/workflows/build-bare-action.yml @@ -0,0 +1,124 @@ +--- +name: Build Bare Onedir + +on: + - workflow_call + +jobs: + + linux: + name: Linux + strategy: + fail-fast: false + matrix: + arch: + - x86_64 + - aarch64 + runs-on: + - self-hosted + - linux + - ${{ matrix.arch }} + steps: + - uses: actions/checkout@v3 + - name: Setup Relenv + uses: ./.github/actions/setup-relenv + with: + platform: linux + arch: ${{ matrix.arch }} + - name: Setup Bare Relenv Onedir + uses: ./.github/actions/build-onedir-bare + with: + platform: linux + arch: ${{ matrix.arch }} + + - name: Set Exit Status + if: always() + run: | + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" + echo "${{ job.status }}" > exitstatus/${{ github.job }}-linux-${{ matrix.arch }} + + - name: Upload Exit Status + if: always() + uses: actions/upload-artifact@v3 + with: + name: exitstatus + path: exitstatus + if-no-files-found: error + + windows: + name: Windows + strategy: + fail-fast: false + matrix: + arch: + - amd64 + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Setup Relenv + uses: ./.github/actions/setup-relenv + with: + platform: windows + arch: ${{ matrix.arch }} + - name: Setup Bare Relenv Onedir + uses: ./.github/actions/build-onedir-bare + with: + platform: windows + arch: ${{ matrix.arch }} + + - name: Set Exit Status + if: always() + run: | + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" + echo "${{ job.status }}" > exitstatus/${{ github.job }}-windows-${{ matrix.arch }} + + - name: Upload Exit Status + if: always() + uses: actions/upload-artifact@v3 + with: + name: exitstatus + path: exitstatus + if-no-files-found: error + + macos: + name: macOS + strategy: + fail-fast: false + matrix: + arch: + - x86_64 + runs-on: macos-12 + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Setup Relenv + uses: ./.github/actions/setup-relenv + with: + platform: darwin + arch: ${{ matrix.arch }} + - name: Setup Bare Relenv Onedir + uses: ./.github/actions/build-onedir-bare + with: + platform: darwin + arch: ${{ matrix.arch }} + + - name: Set Exit Status + if: always() + run: | + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" + echo "${{ job.status }}" > exitstatus/${{ github.job }}-macos-${{ matrix.arch }} + + - name: Upload Exit Status + if: always() + uses: actions/upload-artifact@v3 + with: + name: exitstatus + path: exitstatus + if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e5f97d4af2..63ba067a086 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -241,15 +241,273 @@ jobs: with: changed-files: ${{ needs.prepare-ci.outputs.changed-files }} + build-bare-linux: + name: Build Bare Onedir Linux + if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-ci + strategy: + fail-fast: false + matrix: + arch: + - x86_64 + - aarch64 + runs-on: + - self-hosted + - linux + - ${{ matrix.arch }} + steps: + - uses: actions/checkout@v3 + - name: Setup Relenv + uses: ./.github/actions/setup-relenv + with: + platform: linux + arch: ${{ matrix.arch }} + - name: Setup Bare Relenv Onedir + uses: ./.github/actions/build-onedir-bare + with: + platform: linux + arch: ${{ matrix.arch }} + + - name: Set Exit Status + if: always() + run: | + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" + echo "${{ job.status }}" > exitstatus/${{ github.job }}-linux-${{ matrix.arch }} + + - name: Upload Exit Status + if: always() + uses: actions/upload-artifact@v3 + with: + name: exitstatus + path: exitstatus + if-no-files-found: error + + build-salt-linux: + name: Build Salt Onedir Linux + if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-ci + - build-bare-linux + strategy: + fail-fast: false + matrix: + arch: + - x86_64 + - aarch64 + runs-on: + - self-hosted + - linux + - ${{ matrix.arch }} + steps: + - uses: actions/checkout@v3 + - name: Setup Relenv + uses: ./.github/actions/setup-relenv + with: + platform: linux + arch: ${{ matrix.arch }} + - name: Install Salt into Relenv Onedir + uses: ./.github/actions/build-onedir-pkg + with: + platform: linux + arch: ${{ matrix.arch }} + + - name: Set Exit Status + if: always() + run: | + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" + echo "${{ job.status }}" > exitstatus/${{ github.job }}-linux-${{ matrix.arch }} + + - name: Upload Exit Status + if: always() + uses: actions/upload-artifact@v3 + with: + name: exitstatus + path: exitstatus + if-no-files-found: error + + build-bare-windows: + name: Build Bare Onedir Windows + if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['github-hosted-runners'] }} + needs: + - prepare-ci + strategy: + fail-fast: false + matrix: + arch: + - amd64 + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Setup Relenv + uses: ./.github/actions/setup-relenv + with: + platform: windows + arch: ${{ matrix.arch }} + - name: Setup Bare Relenv Onedir + uses: ./.github/actions/build-onedir-bare + with: + platform: windows + arch: ${{ matrix.arch }} + + - name: Set Exit Status + if: always() + run: | + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" + echo "${{ job.status }}" > exitstatus/${{ github.job }}-windows-${{ matrix.arch }} + + - name: Upload Exit Status + if: always() + uses: actions/upload-artifact@v3 + with: + name: exitstatus + path: exitstatus + if-no-files-found: error + + build-salt-windows: + name: Build Salt Onedir Windows + if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['github-hosted-runners'] }} + needs: + - prepare-ci + - build-bare-windows + strategy: + fail-fast: false + matrix: + arch: + - amd64 + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Setup Relenv + uses: ./.github/actions/setup-relenv + with: + platform: windows + arch: ${{ matrix.arch }} + - name: Install Salt into Relenv Onedir + uses: ./.github/actions/build-onedir-pkg + with: + platform: windows + arch: ${{ matrix.arch }} + + - name: Set Exit Status + if: always() + run: | + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" + echo "${{ job.status }}" > exitstatus/${{ github.job }}-windows-${{ matrix.arch }} + + - name: Upload Exit Status + if: always() + uses: actions/upload-artifact@v3 + with: + name: exitstatus + path: exitstatus + if-no-files-found: error + + build-bare-macos: + name: Build Bare Onedir macOS + if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['github-hosted-runners'] }} + needs: + - prepare-ci + strategy: + fail-fast: false + matrix: + arch: + - x86_64 + runs-on: macos-12 + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Setup Relenv + uses: ./.github/actions/setup-relenv + with: + platform: darwin + arch: ${{ matrix.arch }} + - name: Setup Bare Relenv Onedir + uses: ./.github/actions/build-onedir-bare + with: + platform: darwin + arch: ${{ matrix.arch }} + + - name: Set Exit Status + if: always() + run: | + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" + echo "${{ job.status }}" > exitstatus/${{ github.job }}-macos-${{ matrix.arch }} + + - name: Upload Exit Status + if: always() + uses: actions/upload-artifact@v3 + with: + name: exitstatus + path: exitstatus + if-no-files-found: error + + build-salt-macos: + name: Build Salt Onedir macOS + if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['github-hosted-runners'] }} + needs: + - prepare-ci + - build-bare-macos + strategy: + fail-fast: false + matrix: + arch: + - x86_64 + runs-on: macos-12 + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Setup Relenv + uses: ./.github/actions/setup-relenv + with: + platform: darwin + arch: ${{ matrix.arch }} + - name: Install Salt into Relenv Onedir + uses: ./.github/actions/build-onedir-pkg + with: + platform: darwin + arch: ${{ matrix.arch }} + + - name: Set Exit Status + if: always() + run: | + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" + echo "${{ job.status }}" > exitstatus/${{ github.job }}-macos-${{ matrix.arch }} + + - name: Upload Exit Status + if: always() + uses: actions/upload-artifact@v3 + with: + name: exitstatus + path: exitstatus + if-no-files-found: error + windows-2016: name: Windows 2016 if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-windows uses: ./.github/workflows/test-action.yml with: distro-slug: windows-2016 - nox-session: ci-test + nox-session: ci-test-onedir + platform: windows + arch: amd64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -259,10 +517,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-windows uses: ./.github/workflows/test-action.yml with: distro-slug: windows-2019 - nox-session: ci-test + nox-session: ci-test-onedir + platform: windows + arch: amd64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -272,10 +533,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-windows uses: ./.github/workflows/test-action.yml with: distro-slug: windows-2022 - nox-session: ci-test + nox-session: ci-test-onedir + platform: windows + arch: amd64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -285,10 +549,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['github-hosted-runners'] }} needs: - prepare-ci + - build-salt-macos uses: ./.github/workflows/test-action-macos.yml with: distro-slug: macos-12 - nox-session: ci-test + nox-session: ci-test-onedir + platform: darwin + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -298,10 +565,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: almalinux-8 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -311,10 +581,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: almalinux-9 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -324,10 +597,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: amazonlinux-2 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -337,10 +613,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: archlinux-lts - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -350,10 +629,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: centos-7 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -363,10 +645,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: centosstream-8 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -376,10 +661,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: centosstream-9 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -389,10 +677,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: debian-10 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -402,23 +693,29 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: debian-11 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} debian-11-arm64: - name: Debian 11 ARM64 + name: Debian 11 Arm64 if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: debian-11-arm64 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: aarch64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -428,10 +725,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: fedora-36 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -441,10 +741,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: opensuse-15 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -454,10 +757,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: photonos-3 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -467,10 +773,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: photonos-4 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -480,10 +789,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: ubuntu-18.04 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -493,10 +805,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: ubuntu-20.04 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -506,10 +821,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: ubuntu-20.04-arm64 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: aarch64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -519,10 +837,13 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: ubuntu-22.04 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: x86_64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} @@ -532,15 +853,17 @@ jobs: if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }} needs: - prepare-ci + - build-salt-linux uses: ./.github/workflows/test-action.yml with: distro-slug: ubuntu-22.04-arm64 - nox-session: ci-test + nox-session: ci-test-onedir + platform: linux + arch: aarch64 testrun: ${{ needs.prepare-ci.outputs.testrun }} salt-version: "${{ needs.prepare-ci.outputs.salt-version }}" cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }} - set-pipeline-exit-status: # This step is just so we can make github require this step, to pass checks # on a pull request instead of requiring all diff --git a/.github/workflows/dependencies-action.yml b/.github/workflows/dependencies-action.yml new file mode 100644 index 00000000000..e83c1e16ac5 --- /dev/null +++ b/.github/workflows/dependencies-action.yml @@ -0,0 +1,109 @@ +name: Test Artifact + +on: + - workflow_call + +jobs: + + linux: + name: Linux + strategy: + fail-fast: false + matrix: + arch: + - x86_64 + - aarch64 + runs-on: + - self-hosted + - linux + - ${{ matrix.arch }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - uses: ./.github/actions/build-onedir-pkg + with: + platform: linux + arch: ${{ matrix.arch }} + + - name: Set Exit Status + if: always() + run: | + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" + echo "${{ job.status }}" > exitstatus/${{ github.job }}-linux-${{ matrix.arch }} + + - name: Upload Exit Status + if: always() + uses: actions/upload-artifact@v3 + with: + name: exitstatus + path: exitstatus + if-no-files-found: error + + windows: + name: Windows + strategy: + fail-fast: false + matrix: + arch: + - amd64 + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - uses: ./.github/actions/build-onedir-pkg + with: + platform: windows + arch: ${{ matrix.arch }} + + - name: Set Exit Status + if: always() + run: | + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" + echo "${{ job.status }}" > exitstatus/${{ github.job }}-windows-${{ matrix.arch }} + + - name: Upload Exit Status + if: always() + uses: actions/upload-artifact@v3 + with: + name: exitstatus + path: exitstatus + if-no-files-found: error + + macos: + name: macOS + strategy: + fail-fast: false + matrix: + arch: + - x86_64 + runs-on: macos-12 + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - uses: ./.github/actions/build-onedir-pkg + with: + platform: darwin + arch: ${{ matrix.arch }} + + - name: Set Exit Status + if: always() + run: | + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" + echo "${{ job.status }}" > exitstatus/${{ github.job }}-macos-${{ matrix.arch }} + + - name: Upload Exit Status + if: always() + uses: actions/upload-artifact@v3 + with: + name: exitstatus + path: exitstatus + if-no-files-found: error diff --git a/.github/workflows/docs-action.yml b/.github/workflows/docs-action.yml index e065828d17c..33f3c50513c 100644 --- a/.github/workflows/docs-action.yml +++ b/.github/workflows/docs-action.yml @@ -58,7 +58,7 @@ jobs: - name: Set Exit Status if: always() run: | - mkdir exitstatus + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" echo "${{ job.status }}" > exitstatus/${{ github.job }}-docs-html - name: Upload Exit Status @@ -120,7 +120,7 @@ jobs: - name: Set Exit Status if: always() run: | - mkdir exitstatus + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" echo "${{ job.status }}" > exitstatus/${{ github.job }}-docs-man - name: Upload Exit Status diff --git a/.github/workflows/lint-action.yml b/.github/workflows/lint-action.yml index 31a8eab9e2d..dc0e10f91a8 100644 --- a/.github/workflows/lint-action.yml +++ b/.github/workflows/lint-action.yml @@ -57,7 +57,7 @@ jobs: - name: Set Exit Status if: always() run: | - mkdir exitstatus + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" echo "${{ job.status }}" > exitstatus/${{ github.job }}-lint-salt - name: Upload Exit Status @@ -116,7 +116,7 @@ jobs: - name: Set Exit Status if: always() run: | - mkdir exitstatus + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" echo "${{ job.status }}" > exitstatus/${{ github.job }}-lint-salt - name: Upload Exit Status diff --git a/.github/workflows/pre-commit-action.yml b/.github/workflows/pre-commit-action.yml index ca7a2a70e60..679c0668f4d 100644 --- a/.github/workflows/pre-commit-action.yml +++ b/.github/workflows/pre-commit-action.yml @@ -61,7 +61,7 @@ jobs: - name: Set Exit Status if: always() run: | - mkdir exitstatus + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" echo "${{ job.status }}" > exitstatus/${{ github.job }}-pre-commit - name: Upload Exit Status diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index 6a3733a9348..da734748546 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -15,10 +15,6 @@ on: required: true type: string description: JSON string containing information about what and how to run the test suite - cache-seed: - required: true - type: string - description: Seed used to invalidate caches python-version: required: false type: string @@ -28,6 +24,23 @@ on: type: string required: true description: The Salt version to set prior to running tests. + cache-seed: + required: true + type: string + description: Seed used to invalidate caches + platform: + required: true + type: string + description: The platform being tested + arch: + required: true + type: string + description: The platform arch being tested + package-name: + required: false + type: string + description: The onedir package name to use + default: salt env: @@ -86,6 +99,21 @@ jobs: path: nox.${{ inputs.distro-slug }}.tar.* key: ${{ inputs.cache-seed }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ matrix.transport }}|${{ inputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }} + - name: Download Onedir Tarball as an Artifact + if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz + path: artifacts/ + + - name: Decompress Onedir Tarball + if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' + shell: bash + run: | + python3 -c "import os; os.makedirs('artifacts', exist_ok=True)" + cd artifacts + tar xvf ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz + # Skip jobs if nox.*.tar.* is already cached - name: Set up Python ${{ inputs.python-version }} if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' @@ -110,9 +138,9 @@ jobs: if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' run: | if [ "${{ matrix.transport }}" != "tcp" ]; then - echo NOX_SESSION=${{ inputs.nox-session}}-3 >> $GITHUB_ENV + echo NOX_SESSION=${{ inputs.nox-session}} >> $GITHUB_ENV else - echo NOX_SESSION=${{ inputs.nox-session}}-tcp-3 >> $GITHUB_ENV + echo NOX_SESSION=${{ inputs.nox-session}}-tcp >> $GITHUB_ENV fi - name: Install Dependencies @@ -142,7 +170,7 @@ jobs: - name: Set Exit Status if: always() run: | - mkdir exitstatus + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" echo "${{ job.status }}" > exitstatus/${{ github.job }}-${{ inputs.distro-slug }}-deps - name: Upload Exit Status @@ -173,6 +201,19 @@ jobs: run: | echo "${{ inputs.salt-version }}" > salt/_version.txt + - name: Download Onedir Tarball as an Artifact + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz + path: artifacts/ + + - name: Decompress Onedir Tarball + shell: bash + run: | + python3 -c "import os; os.makedirs('artifacts', exist_ok=True)" + cd artifacts + tar xvf ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz + - name: Install System Dependencies if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' run: | @@ -200,9 +241,9 @@ jobs: - name: Define Nox Session run: | if [ "${{ matrix.transport }}" != "tcp" ]; then - echo NOX_SESSION=${{ inputs.nox-session}}-3 >> $GITHUB_ENV + echo NOX_SESSION=${{ inputs.nox-session}} >> $GITHUB_ENV else - echo NOX_SESSION=${{ inputs.nox-session}}-tcp-3 >> $GITHUB_ENV + echo NOX_SESSION=${{ inputs.nox-session}}-tcp >> $GITHUB_ENV fi - name: Download testrun-changed-files.txt @@ -320,7 +361,7 @@ jobs: - name: Set Exit Status if: always() run: | - mkdir exitstatus + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" echo "${{ job.status }}" > exitstatus/${{ github.job }}-${{ inputs.distro-slug }}-${{ env.NOX_SESSION }}-${{ matrix.tests-chunk }}-tests - name: Upload Exit Status @@ -350,9 +391,9 @@ jobs: - name: Define Nox Session run: | if [ "${{ matrix.transport }}" != "tcp" ]; then - echo NOX_SESSION=${{ inputs.nox-session}}-3 >> $GITHUB_ENV + echo NOX_SESSION=${{ inputs.nox-session}} >> $GITHUB_ENV else - echo NOX_SESSION=${{ inputs.nox-session}}-tcp-3 >> $GITHUB_ENV + echo NOX_SESSION=${{ inputs.nox-session}}-tcp >> $GITHUB_ENV fi - name: Download Test Run Artifacts @@ -374,10 +415,10 @@ jobs: name: code-coverage path: artifacts/coverage - - name: Set up Python 3.10 + - name: Set up Python 3.9 uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.9" - name: Install Nox run: | @@ -410,7 +451,7 @@ jobs: - name: Set Exit Status if: always() run: | - mkdir exitstatus + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" echo "${{ job.status }}" > exitstatus/${{ github.job }}-${{ inputs.distro-slug }}-${{ env.NOX_SESSION }}-report - name: Upload Exit Status diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index 6b415f4221d..1e3ffde4b4d 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -23,6 +23,19 @@ on: required: true type: string description: Seed used to invalidate caches + platform: + required: true + type: string + description: The platform being tested + arch: + required: true + type: string + description: The platform arch being tested + package-name: + required: false + type: string + description: The onedir package name to use + default: salt env: @@ -87,6 +100,21 @@ jobs: path: nox.${{ inputs.distro-slug }}.tar.* key: ${{ inputs.cache-seed }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ matrix.transport }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }} + - name: Download Onedir Tarball as an Artifact + if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz + path: artifacts/ + + - name: Decompress Onedir Tarball + if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' + shell: bash + run: | + python3 -c "import os; os.makedirs('artifacts', exist_ok=True)" + cd artifacts + tar xvf ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz + # Skip jobs if nox.*.tar.* is already cached - name: PyPi Proxy if: steps.nox-dependencies-cache.outputs.cache-hit != 'true' @@ -99,9 +127,9 @@ jobs: - name: Define Nox Session run: | if [ "${{ matrix.transport }}" != "tcp" ]; then - echo NOX_SESSION=${{ inputs.nox-session}}-3 >> $GITHUB_ENV + echo NOX_SESSION=${{ inputs.nox-session }} >> $GITHUB_ENV else - echo NOX_SESSION=${{ inputs.nox-session}}-tcp-3 >> $GITHUB_ENV + echo NOX_SESSION=${{ inputs.nox-session }}-tcp >> $GITHUB_ENV fi - name: Start VM @@ -147,7 +175,7 @@ jobs: - name: Set Exit Status if: always() run: | - mkdir exitstatus + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" echo "${{ job.status }}" > exitstatus/${{ github.job }}-${{ inputs.distro-slug }}-deps - name: Upload Exit Status @@ -181,6 +209,19 @@ jobs: run: | echo "${{ inputs.salt-version }}" > salt/_version.txt + - name: Download Onedir Tarball as an Artifact + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz + path: artifacts/ + + - name: Decompress Onedir Tarball + shell: bash + run: | + python3 -c "import os; os.makedirs('artifacts', exist_ok=True)" + cd artifacts + tar xvf ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz + - name: Download cached nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }} uses: actions/cache@v3 with: @@ -197,9 +238,9 @@ jobs: - name: Define Nox Session run: | if [ "${{ matrix.transport }}" != "tcp" ]; then - echo NOX_SESSION=${{ inputs.nox-session}}-3 >> $GITHUB_ENV + echo NOX_SESSION=${{ inputs.nox-session }} >> $GITHUB_ENV else - echo NOX_SESSION=${{ inputs.nox-session}}-tcp-3 >> $GITHUB_ENV + echo NOX_SESSION=${{ inputs.nox-session }}-tcp >> $GITHUB_ENV fi - name: Download testrun-changed-files.txt @@ -321,7 +362,7 @@ jobs: - name: Set Exit Status if: always() run: | - mkdir exitstatus + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" echo "${{ job.status }}" > exitstatus/${{ github.job }}-${{ inputs.distro-slug }}-${{ env.NOX_SESSION }}-${{ matrix.tests-chunk }}-tests - name: Upload Exit Status @@ -354,9 +395,9 @@ jobs: - name: Define Nox Session run: | if [ "${{ matrix.transport }}" != "tcp" ]; then - echo NOX_SESSION=${{ inputs.nox-session}}-3 >> $GITHUB_ENV + echo NOX_SESSION=${{ inputs.nox-session }} >> $GITHUB_ENV else - echo NOX_SESSION=${{ inputs.nox-session}}-tcp-3 >> $GITHUB_ENV + echo NOX_SESSION=${{ inputs.nox-session }}-tcp >> $GITHUB_ENV fi - name: Download Test Run Artifacts @@ -378,10 +419,10 @@ jobs: name: code-coverage path: artifacts/coverage - - name: Set up Python 3.10 + - name: Set up Python 3.9 uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.9" - name: Install Nox run: | @@ -414,7 +455,7 @@ jobs: - name: Set Exit Status if: always() run: | - mkdir exitstatus + python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)" echo "${{ job.status }}" > exitstatus/${{ github.job }}-${{ inputs.distro-slug }}-${{ env.NOX_SESSION }}-report - name: Upload Exit Status diff --git a/noxfile.py b/noxfile.py index 8cdad5283c0..05367f76c30 100644 --- a/noxfile.py +++ b/noxfile.py @@ -69,14 +69,17 @@ if COVERAGE_FILE is None: IS_DARWIN = sys.platform.lower().startswith("darwin") IS_WINDOWS = sys.platform.lower().startswith("win") IS_FREEBSD = sys.platform.lower().startswith("freebsd") +ONEDIR_ARTIFACT_PATH = ARTIFACTS_DIR / "salt" +if IS_WINDOWS: + ONEDIR_PYTHON_PATH = ONEDIR_ARTIFACT_PATH / "Scripts" / "python.exe" +else: + ONEDIR_PYTHON_PATH = ONEDIR_ARTIFACT_PATH / "bin" / "python3" # Python versions to run against _PYTHON_VERSIONS = ("3", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10") # Nox options # Reuse existing virtualenvs nox.options.reuse_existing_virtualenvs = True -# Don't fail on missing interpreters -nox.options.error_on_missing_interpreters = False # Change current directory to REPO_ROOT os.chdir(str(REPO_ROOT)) @@ -117,8 +120,11 @@ def session_run_always(session, *command, **kwargs): session._runner.global_config.install_only = old_install_only_value -def find_session_runner(session, name, python_version, **kwargs): - name += "-{}".format(python_version) +def find_session_runner(session, name, python_version, onedir=False, **kwargs): + if onedir: + name += "-onedir-{}".format(ONEDIR_PYTHON_PATH) + else: + name += "-{}".format(python_version) for s, _ in session._runner.manifest.list_all_sessions(): if name not in s.signatures: continue @@ -301,11 +307,18 @@ def _upgrade_pip_setuptools_and_wheel(session, upgrade=True): def _install_requirements( - session, transport, *extra_requirements, requirements_type="ci" + session, + transport, + *extra_requirements, + requirements_type="ci", + onedir=False, ): if not _upgrade_pip_setuptools_and_wheel(session): return False + if onedir and not IS_WINDOWS and not IS_DARWIN and not IS_FREEBSD: + session_run_always(session, "python3", "-m", "relenv", "toolchain", "fetch") + # Install requirements requirements_file = _get_pip_requirements_file( session, transport, requirements_type=requirements_type @@ -1022,9 +1035,12 @@ def _pytest(session, coverage, cmd_args, env=None): session.run("python", "-m", "pytest", *args, env=env) -def _ci_test(session, transport): +def _ci_test(session, transport, onedir=False): # Install requirements - _install_requirements(session, transport) + _install_requirements(session, transport, onedir=onedir) + env = { + "ONEDIR_TESTRUN": "1", + } chunks = { "unit": [ "tests/unit", @@ -1086,7 +1102,7 @@ def _ci_test(session, transport): ] + chunk_cmd ) - _pytest(session, track_code_coverage, pytest_args) + _pytest(session, track_code_coverage, pytest_args, env=env) except CommandFailed: if rerun_failures is False: raise @@ -1106,7 +1122,7 @@ def _ci_test(session, transport): ] + chunk_cmd ) - _pytest(session, track_code_coverage, pytest_args) + _pytest(session, track_code_coverage, pytest_args, env=env) @nox.session(python=_PYTHON_VERSIONS, name="ci-test") @@ -1119,6 +1135,38 @@ def ci_test_tcp(session): _ci_test(session, "tcp") +@nox.session( + python=str(ONEDIR_PYTHON_PATH), + name="ci-test-onedir", + venv_params=["--system-site-packages"], +) +def ci_test_onedir(session): + if not ONEDIR_ARTIFACT_PATH.exists(): + session.error( + "The salt onedir artifact, expected to be in '{}', was not found".format( + ONEDIR_ARTIFACT_PATH.relative_to(REPO_ROOT) + ) + ) + + _ci_test(session, "zeromq", onedir=True) + + +@nox.session( + python=str(ONEDIR_PYTHON_PATH), + name="ci-test-onedir-tcp", + venv_params=["--system-site-packages"], +) +def ci_test_onedir_tcp(session): + if not ONEDIR_ARTIFACT_PATH.exists(): + session.error( + "The salt onedir artifact, expected to be in '{}', was not found".format( + ONEDIR_ARTIFACT_PATH.relative_to(REPO_ROOT) + ) + ) + + _ci_test(session, "tcp", onedir=True) + + @nox.session(python="3", name="report-coverage") def report_coverage(session): _report_coverage(session) diff --git a/tools/vm.py b/tools/vm.py index 9c7cf511e26..fbe440a5647 100644 --- a/tools/vm.py +++ b/tools/vm.py @@ -976,11 +976,17 @@ class VM: "--exclude", ".pytest_cache/", "--exclude", - "artifacts/", - "--exclude", f"{STATE_DIR.relative_to(REPO_ROOT)}{os.path.sep}", "--exclude", "*.py~", + # We need to include artifacts/ to be able to include artifacts/salt + "--include", + "artifacts/", + "--include", + "artifacts/salt", + # But we also want to exclude all other entries under artifacts/ + "--exclude", + "artifacts/*", ] if self.is_windows: # Symlinks aren't handled properly on windows, just replace the