Start building onedir packages

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>
This commit is contained in:
Pedro Algarvio 2023-01-17 04:50:55 +00:00 committed by Pedro Algarvio
parent 2ca75aaed5
commit 2e4314b30f
14 changed files with 917 additions and 67 deletions

View file

@ -32,6 +32,7 @@ ignore_errors = True
[paths]
salt =
salt/
artifacts/salt
**/testing/salt/
**\testing\salt
tests =

View file

@ -0,0 +1,36 @@
---
name: build-onedir-bare
description: Build Bare Onedir Package
inputs:
platform:
required: true
type: string
description: The platform to build
arch:
required: true
type: string
description: The platform arch to build
package-name:
required: false
type: string
description: The onedir package name to create
default: salt
runs:
using: composite
steps:
- name: Cache Bare Onedir Package Directory
id: onedir-pkg-cache
uses: actions/cache@v3
with:
path: artifacts/${{ inputs.package-name }}
key: relenv|${{ env.RELENV_VERSION }}|bare|${{ inputs.platform }}|${{ inputs.arch }}|${{ inputs.package-name }}|${{ hashFiles('.relenv/**/*.xz') }}
- name: Create Onedir Directory
shell: bash
if: steps.onedir-pkg-cache.outputs.cache-hit != 'true'
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
python3 -m relenv create --arch=${{ inputs.arch }} artifacts/${{ inputs.package-name }}

View file

@ -0,0 +1,75 @@
---
name: build-linux-onedir
description: Build Linux Onedir Package
inputs:
platform:
required: true
type: string
description: The platform to build
arch:
required: true
type: string
description: The platform arch to build
package-name:
required: false
type: string
description: The onedir package name to create
default: salt
runs:
using: composite
steps:
- name: Download Cached Bare Onedir Package Directory
id: onedir-bare-cache
uses: actions/cache@v3
with:
path: artifacts/${{ inputs.package-name }}
key: relenv|${{ env.RELENV_VERSION }}|bare|${{ inputs.platform }}|${{ inputs.arch }}|${{ inputs.package-name }}|${{ hashFiles('.relenv/**/*.xz') }}
- name: Cache Pip Wheels
uses: actions/cache@v3
with:
path: .pip-download-cache
key: pip-download-cache|${{ env.RELENV_VERSION }}|salt|${{ inputs.platform }}|${{ inputs.arch }}|${{ inputs.package-name }}|${{ hashFiles('.relenv/**/*.xz', 'requirements/static/pkg/*/*.txt') }}
- name: Install Salt Onedir Package Dependencies
env:
PIP_DOWNLOAD_CACHE: .pip-download-cache
PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
PIP_EXTRA_INDEX_URL: https://pypi.org/simple
shell: bash
run: |
python3 -c "import os; os.makedirs('.downloaded-dependencies', exist_ok=True)"
if [ "${{ inputs.platform }}" != "windows" ]; then
artifacts/${{ inputs.package-name }}/bin/python3 -m pip install -r requirements/static/pkg/py3.10/${{ inputs.platform }}.txt
else
artifacts/${{ inputs.package-name }}/Scripts/python -m pip install -r requirements/static/pkg/py3.10/${{ inputs.platform }}.txt
fi
- name: Install Salt Into Onedir
env:
PIP_DOWNLOAD_CACHE: .pip-download-cache
PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
PIP_EXTRA_INDEX_URL: https://pypi.org/simple
shell: bash
run: |
if [ "${{ inputs.platform }}" != "windows" ]; then
artifacts/${{ inputs.package-name }}/bin/python3 -m pip install .
else
artifacts/${{ inputs.package-name }}/Scripts/python -m pip install .
fi
- name: Create Tarball
shell: bash
run: |
cd artifacts/
tar -cJf ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz ${{ inputs.package-name }}
- name: Upload Onedir Tarball as an Artifact
uses: actions/upload-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz
path: artifacts/${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz
retention-days: 7

46
.github/actions/setup-relenv/action.yml vendored Normal file
View file

@ -0,0 +1,46 @@
---
name: setup-relenv
description: Setup Relenv
inputs:
platform:
required: true
type: string
description: The platform to build
arch:
required: true
type: string
description: The platform arch to build
runs:
using: composite
steps:
- name: Install Relenv
shell: bash
run: |
python3 -m pip install relenv
- name: Set environment variables
shell: bash
run: |
python3 -c "import os; os.makedirs('.relenv', exist_ok=True)"
echo RELENV_DATA="$(pwd)/.relenv" >> $GITHUB_ENV
echo RELENV_VERSION="$(python3 -m pip show relenv | grep Version | cut -d ' ' -f2)" >> $GITHUB_ENV
- name: Cache Relenv Data Directory
uses: actions/cache@v3
with:
path: ${{ env.RELENV_DATA }}
key: relenv|${{ env.RELENV_VERSION }}|${{ inputs.platform }}|${{ inputs.arch }} }}
- name: Fetch Toolchain
if: ${{ inputs.platform == 'linux' }}
shell: bash
run: |
python3 -m relenv toolchain fetch --arch=${{ inputs.arch }}
- name: Fetch Native Python Build
shell: bash
run: |
python3 -m relenv fetch --arch=${{ inputs.arch }}

124
.github/workflows/build-bare-action.yml vendored Normal file
View file

@ -0,0 +1,124 @@
---
name: Build Bare Onedir
on:
- workflow_call
jobs:
linux:
name: Linux
strategy:
fail-fast: false
matrix:
arch:
- x86_64
- aarch64
runs-on:
- self-hosted
- linux
- ${{ matrix.arch }}
steps:
- uses: actions/checkout@v3
- name: Setup Relenv
uses: ./.github/actions/setup-relenv
with:
platform: linux
arch: ${{ matrix.arch }}
- name: Setup Bare Relenv Onedir
uses: ./.github/actions/build-onedir-bare
with:
platform: linux
arch: ${{ matrix.arch }}
- name: Set Exit Status
if: always()
run: |
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-linux-${{ matrix.arch }}
- name: Upload Exit Status
if: always()
uses: actions/upload-artifact@v3
with:
name: exitstatus
path: exitstatus
if-no-files-found: error
windows:
name: Windows
strategy:
fail-fast: false
matrix:
arch:
- amd64
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Setup Relenv
uses: ./.github/actions/setup-relenv
with:
platform: windows
arch: ${{ matrix.arch }}
- name: Setup Bare Relenv Onedir
uses: ./.github/actions/build-onedir-bare
with:
platform: windows
arch: ${{ matrix.arch }}
- name: Set Exit Status
if: always()
run: |
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-windows-${{ matrix.arch }}
- name: Upload Exit Status
if: always()
uses: actions/upload-artifact@v3
with:
name: exitstatus
path: exitstatus
if-no-files-found: error
macos:
name: macOS
strategy:
fail-fast: false
matrix:
arch:
- x86_64
runs-on: macos-12
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Setup Relenv
uses: ./.github/actions/setup-relenv
with:
platform: darwin
arch: ${{ matrix.arch }}
- name: Setup Bare Relenv Onedir
uses: ./.github/actions/build-onedir-bare
with:
platform: darwin
arch: ${{ matrix.arch }}
- name: Set Exit Status
if: always()
run: |
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-macos-${{ matrix.arch }}
- name: Upload Exit Status
if: always()
uses: actions/upload-artifact@v3
with:
name: exitstatus
path: exitstatus
if-no-files-found: error

View file

@ -241,15 +241,273 @@ jobs:
with:
changed-files: ${{ needs.prepare-ci.outputs.changed-files }}
build-bare-linux:
name: Build Bare Onedir Linux
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
strategy:
fail-fast: false
matrix:
arch:
- x86_64
- aarch64
runs-on:
- self-hosted
- linux
- ${{ matrix.arch }}
steps:
- uses: actions/checkout@v3
- name: Setup Relenv
uses: ./.github/actions/setup-relenv
with:
platform: linux
arch: ${{ matrix.arch }}
- name: Setup Bare Relenv Onedir
uses: ./.github/actions/build-onedir-bare
with:
platform: linux
arch: ${{ matrix.arch }}
- name: Set Exit Status
if: always()
run: |
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-linux-${{ matrix.arch }}
- name: Upload Exit Status
if: always()
uses: actions/upload-artifact@v3
with:
name: exitstatus
path: exitstatus
if-no-files-found: error
build-salt-linux:
name: Build Salt Onedir Linux
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-bare-linux
strategy:
fail-fast: false
matrix:
arch:
- x86_64
- aarch64
runs-on:
- self-hosted
- linux
- ${{ matrix.arch }}
steps:
- uses: actions/checkout@v3
- name: Setup Relenv
uses: ./.github/actions/setup-relenv
with:
platform: linux
arch: ${{ matrix.arch }}
- name: Install Salt into Relenv Onedir
uses: ./.github/actions/build-onedir-pkg
with:
platform: linux
arch: ${{ matrix.arch }}
- name: Set Exit Status
if: always()
run: |
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-linux-${{ matrix.arch }}
- name: Upload Exit Status
if: always()
uses: actions/upload-artifact@v3
with:
name: exitstatus
path: exitstatus
if-no-files-found: error
build-bare-windows:
name: Build Bare Onedir Windows
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['github-hosted-runners'] }}
needs:
- prepare-ci
strategy:
fail-fast: false
matrix:
arch:
- amd64
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Setup Relenv
uses: ./.github/actions/setup-relenv
with:
platform: windows
arch: ${{ matrix.arch }}
- name: Setup Bare Relenv Onedir
uses: ./.github/actions/build-onedir-bare
with:
platform: windows
arch: ${{ matrix.arch }}
- name: Set Exit Status
if: always()
run: |
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-windows-${{ matrix.arch }}
- name: Upload Exit Status
if: always()
uses: actions/upload-artifact@v3
with:
name: exitstatus
path: exitstatus
if-no-files-found: error
build-salt-windows:
name: Build Salt Onedir Windows
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['github-hosted-runners'] }}
needs:
- prepare-ci
- build-bare-windows
strategy:
fail-fast: false
matrix:
arch:
- amd64
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Setup Relenv
uses: ./.github/actions/setup-relenv
with:
platform: windows
arch: ${{ matrix.arch }}
- name: Install Salt into Relenv Onedir
uses: ./.github/actions/build-onedir-pkg
with:
platform: windows
arch: ${{ matrix.arch }}
- name: Set Exit Status
if: always()
run: |
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-windows-${{ matrix.arch }}
- name: Upload Exit Status
if: always()
uses: actions/upload-artifact@v3
with:
name: exitstatus
path: exitstatus
if-no-files-found: error
build-bare-macos:
name: Build Bare Onedir macOS
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['github-hosted-runners'] }}
needs:
- prepare-ci
strategy:
fail-fast: false
matrix:
arch:
- x86_64
runs-on: macos-12
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Setup Relenv
uses: ./.github/actions/setup-relenv
with:
platform: darwin
arch: ${{ matrix.arch }}
- name: Setup Bare Relenv Onedir
uses: ./.github/actions/build-onedir-bare
with:
platform: darwin
arch: ${{ matrix.arch }}
- name: Set Exit Status
if: always()
run: |
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-macos-${{ matrix.arch }}
- name: Upload Exit Status
if: always()
uses: actions/upload-artifact@v3
with:
name: exitstatus
path: exitstatus
if-no-files-found: error
build-salt-macos:
name: Build Salt Onedir macOS
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['github-hosted-runners'] }}
needs:
- prepare-ci
- build-bare-macos
strategy:
fail-fast: false
matrix:
arch:
- x86_64
runs-on: macos-12
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Setup Relenv
uses: ./.github/actions/setup-relenv
with:
platform: darwin
arch: ${{ matrix.arch }}
- name: Install Salt into Relenv Onedir
uses: ./.github/actions/build-onedir-pkg
with:
platform: darwin
arch: ${{ matrix.arch }}
- name: Set Exit Status
if: always()
run: |
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-macos-${{ matrix.arch }}
- name: Upload Exit Status
if: always()
uses: actions/upload-artifact@v3
with:
name: exitstatus
path: exitstatus
if-no-files-found: error
windows-2016:
name: Windows 2016
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-windows
uses: ./.github/workflows/test-action.yml
with:
distro-slug: windows-2016
nox-session: ci-test
nox-session: ci-test-onedir
platform: windows
arch: amd64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -259,10 +517,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-windows
uses: ./.github/workflows/test-action.yml
with:
distro-slug: windows-2019
nox-session: ci-test
nox-session: ci-test-onedir
platform: windows
arch: amd64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -272,10 +533,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-windows
uses: ./.github/workflows/test-action.yml
with:
distro-slug: windows-2022
nox-session: ci-test
nox-session: ci-test-onedir
platform: windows
arch: amd64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -285,10 +549,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['github-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-macos
uses: ./.github/workflows/test-action-macos.yml
with:
distro-slug: macos-12
nox-session: ci-test
nox-session: ci-test-onedir
platform: darwin
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -298,10 +565,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: almalinux-8
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -311,10 +581,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: almalinux-9
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -324,10 +597,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: amazonlinux-2
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -337,10 +613,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: archlinux-lts
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -350,10 +629,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: centos-7
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -363,10 +645,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: centosstream-8
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -376,10 +661,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: centosstream-9
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -389,10 +677,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: debian-10
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -402,23 +693,29 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: debian-11
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
debian-11-arm64:
name: Debian 11 ARM64
name: Debian 11 Arm64
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: debian-11-arm64
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: aarch64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -428,10 +725,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: fedora-36
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -441,10 +741,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: opensuse-15
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -454,10 +757,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: photonos-3
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -467,10 +773,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: photonos-4
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -480,10 +789,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: ubuntu-18.04
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -493,10 +805,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: ubuntu-20.04
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -506,10 +821,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: ubuntu-20.04-arm64
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: aarch64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -519,10 +837,13 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: ubuntu-22.04
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: x86_64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
@ -532,15 +853,17 @@ jobs:
if: ${{ fromJSON(needs.prepare-ci.outputs.jobs)['self-hosted-runners'] }}
needs:
- prepare-ci
- build-salt-linux
uses: ./.github/workflows/test-action.yml
with:
distro-slug: ubuntu-22.04-arm64
nox-session: ci-test
nox-session: ci-test-onedir
platform: linux
arch: aarch64
testrun: ${{ needs.prepare-ci.outputs.testrun }}
salt-version: "${{ needs.prepare-ci.outputs.salt-version }}"
cache-seed: ${{ needs.prepare-ci.outputs.cache-seed }}
set-pipeline-exit-status:
# This step is just so we can make github require this step, to pass checks
# on a pull request instead of requiring all

View file

@ -0,0 +1,109 @@
name: Test Artifact
on:
- workflow_call
jobs:
linux:
name: Linux
strategy:
fail-fast: false
matrix:
arch:
- x86_64
- aarch64
runs-on:
- self-hosted
- linux
- ${{ matrix.arch }}
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v4
with:
python-version: "3.10"
- uses: ./.github/actions/build-onedir-pkg
with:
platform: linux
arch: ${{ matrix.arch }}
- name: Set Exit Status
if: always()
run: |
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-linux-${{ matrix.arch }}
- name: Upload Exit Status
if: always()
uses: actions/upload-artifact@v3
with:
name: exitstatus
path: exitstatus
if-no-files-found: error
windows:
name: Windows
strategy:
fail-fast: false
matrix:
arch:
- amd64
runs-on: windows-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v4
with:
python-version: "3.10"
- uses: ./.github/actions/build-onedir-pkg
with:
platform: windows
arch: ${{ matrix.arch }}
- name: Set Exit Status
if: always()
run: |
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-windows-${{ matrix.arch }}
- name: Upload Exit Status
if: always()
uses: actions/upload-artifact@v3
with:
name: exitstatus
path: exitstatus
if-no-files-found: error
macos:
name: macOS
strategy:
fail-fast: false
matrix:
arch:
- x86_64
runs-on: macos-12
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v4
with:
python-version: "3.10"
- uses: ./.github/actions/build-onedir-pkg
with:
platform: darwin
arch: ${{ matrix.arch }}
- name: Set Exit Status
if: always()
run: |
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-macos-${{ matrix.arch }}
- name: Upload Exit Status
if: always()
uses: actions/upload-artifact@v3
with:
name: exitstatus
path: exitstatus
if-no-files-found: error

View file

@ -58,7 +58,7 @@ jobs:
- name: Set Exit Status
if: always()
run: |
mkdir exitstatus
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-docs-html
- name: Upload Exit Status
@ -120,7 +120,7 @@ jobs:
- name: Set Exit Status
if: always()
run: |
mkdir exitstatus
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-docs-man
- name: Upload Exit Status

View file

@ -57,7 +57,7 @@ jobs:
- name: Set Exit Status
if: always()
run: |
mkdir exitstatus
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-lint-salt
- name: Upload Exit Status
@ -116,7 +116,7 @@ jobs:
- name: Set Exit Status
if: always()
run: |
mkdir exitstatus
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-lint-salt
- name: Upload Exit Status

View file

@ -61,7 +61,7 @@ jobs:
- name: Set Exit Status
if: always()
run: |
mkdir exitstatus
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-pre-commit
- name: Upload Exit Status

View file

@ -15,10 +15,6 @@ on:
required: true
type: string
description: JSON string containing information about what and how to run the test suite
cache-seed:
required: true
type: string
description: Seed used to invalidate caches
python-version:
required: false
type: string
@ -28,6 +24,23 @@ on:
type: string
required: true
description: The Salt version to set prior to running tests.
cache-seed:
required: true
type: string
description: Seed used to invalidate caches
platform:
required: true
type: string
description: The platform being tested
arch:
required: true
type: string
description: The platform arch being tested
package-name:
required: false
type: string
description: The onedir package name to use
default: salt
env:
@ -86,6 +99,21 @@ jobs:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-seed }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ matrix.transport }}|${{ inputs.python-version }}|${{ hashFiles('requirements/**/*.txt') }}
- name: Download Onedir Tarball as an Artifact
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz
path: artifacts/
- name: Decompress Onedir Tarball
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz
# Skip jobs if nox.*.tar.* is already cached
- name: Set up Python ${{ inputs.python-version }}
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
@ -110,9 +138,9 @@ jobs:
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
if [ "${{ matrix.transport }}" != "tcp" ]; then
echo NOX_SESSION=${{ inputs.nox-session}}-3 >> $GITHUB_ENV
echo NOX_SESSION=${{ inputs.nox-session}} >> $GITHUB_ENV
else
echo NOX_SESSION=${{ inputs.nox-session}}-tcp-3 >> $GITHUB_ENV
echo NOX_SESSION=${{ inputs.nox-session}}-tcp >> $GITHUB_ENV
fi
- name: Install Dependencies
@ -142,7 +170,7 @@ jobs:
- name: Set Exit Status
if: always()
run: |
mkdir exitstatus
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-${{ inputs.distro-slug }}-deps
- name: Upload Exit Status
@ -173,6 +201,19 @@ jobs:
run: |
echo "${{ inputs.salt-version }}" > salt/_version.txt
- name: Download Onedir Tarball as an Artifact
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz
path: artifacts/
- name: Decompress Onedir Tarball
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz
- name: Install System Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
@ -200,9 +241,9 @@ jobs:
- name: Define Nox Session
run: |
if [ "${{ matrix.transport }}" != "tcp" ]; then
echo NOX_SESSION=${{ inputs.nox-session}}-3 >> $GITHUB_ENV
echo NOX_SESSION=${{ inputs.nox-session}} >> $GITHUB_ENV
else
echo NOX_SESSION=${{ inputs.nox-session}}-tcp-3 >> $GITHUB_ENV
echo NOX_SESSION=${{ inputs.nox-session}}-tcp >> $GITHUB_ENV
fi
- name: Download testrun-changed-files.txt
@ -320,7 +361,7 @@ jobs:
- name: Set Exit Status
if: always()
run: |
mkdir exitstatus
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-${{ inputs.distro-slug }}-${{ env.NOX_SESSION }}-${{ matrix.tests-chunk }}-tests
- name: Upload Exit Status
@ -350,9 +391,9 @@ jobs:
- name: Define Nox Session
run: |
if [ "${{ matrix.transport }}" != "tcp" ]; then
echo NOX_SESSION=${{ inputs.nox-session}}-3 >> $GITHUB_ENV
echo NOX_SESSION=${{ inputs.nox-session}} >> $GITHUB_ENV
else
echo NOX_SESSION=${{ inputs.nox-session}}-tcp-3 >> $GITHUB_ENV
echo NOX_SESSION=${{ inputs.nox-session}}-tcp >> $GITHUB_ENV
fi
- name: Download Test Run Artifacts
@ -374,10 +415,10 @@ jobs:
name: code-coverage
path: artifacts/coverage
- name: Set up Python 3.10
- name: Set up Python 3.9
uses: actions/setup-python@v4
with:
python-version: "3.10"
python-version: "3.9"
- name: Install Nox
run: |
@ -410,7 +451,7 @@ jobs:
- name: Set Exit Status
if: always()
run: |
mkdir exitstatus
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-${{ inputs.distro-slug }}-${{ env.NOX_SESSION }}-report
- name: Upload Exit Status

View file

@ -23,6 +23,19 @@ on:
required: true
type: string
description: Seed used to invalidate caches
platform:
required: true
type: string
description: The platform being tested
arch:
required: true
type: string
description: The platform arch being tested
package-name:
required: false
type: string
description: The onedir package name to use
default: salt
env:
@ -87,6 +100,21 @@ jobs:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-seed }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ matrix.transport }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json') }}
- name: Download Onedir Tarball as an Artifact
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz
path: artifacts/
- name: Decompress Onedir Tarball
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz
# Skip jobs if nox.*.tar.* is already cached
- name: PyPi Proxy
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
@ -99,9 +127,9 @@ jobs:
- name: Define Nox Session
run: |
if [ "${{ matrix.transport }}" != "tcp" ]; then
echo NOX_SESSION=${{ inputs.nox-session}}-3 >> $GITHUB_ENV
echo NOX_SESSION=${{ inputs.nox-session }} >> $GITHUB_ENV
else
echo NOX_SESSION=${{ inputs.nox-session}}-tcp-3 >> $GITHUB_ENV
echo NOX_SESSION=${{ inputs.nox-session }}-tcp >> $GITHUB_ENV
fi
- name: Start VM
@ -147,7 +175,7 @@ jobs:
- name: Set Exit Status
if: always()
run: |
mkdir exitstatus
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-${{ inputs.distro-slug }}-deps
- name: Upload Exit Status
@ -181,6 +209,19 @@ jobs:
run: |
echo "${{ inputs.salt-version }}" > salt/_version.txt
- name: Download Onedir Tarball as an Artifact
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz
path: artifacts/
- name: Decompress Onedir Tarball
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.arch }}-${{ inputs.platform }}.tar.xz
- name: Download cached nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
uses: actions/cache@v3
with:
@ -197,9 +238,9 @@ jobs:
- name: Define Nox Session
run: |
if [ "${{ matrix.transport }}" != "tcp" ]; then
echo NOX_SESSION=${{ inputs.nox-session}}-3 >> $GITHUB_ENV
echo NOX_SESSION=${{ inputs.nox-session }} >> $GITHUB_ENV
else
echo NOX_SESSION=${{ inputs.nox-session}}-tcp-3 >> $GITHUB_ENV
echo NOX_SESSION=${{ inputs.nox-session }}-tcp >> $GITHUB_ENV
fi
- name: Download testrun-changed-files.txt
@ -321,7 +362,7 @@ jobs:
- name: Set Exit Status
if: always()
run: |
mkdir exitstatus
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-${{ inputs.distro-slug }}-${{ env.NOX_SESSION }}-${{ matrix.tests-chunk }}-tests
- name: Upload Exit Status
@ -354,9 +395,9 @@ jobs:
- name: Define Nox Session
run: |
if [ "${{ matrix.transport }}" != "tcp" ]; then
echo NOX_SESSION=${{ inputs.nox-session}}-3 >> $GITHUB_ENV
echo NOX_SESSION=${{ inputs.nox-session }} >> $GITHUB_ENV
else
echo NOX_SESSION=${{ inputs.nox-session}}-tcp-3 >> $GITHUB_ENV
echo NOX_SESSION=${{ inputs.nox-session }}-tcp >> $GITHUB_ENV
fi
- name: Download Test Run Artifacts
@ -378,10 +419,10 @@ jobs:
name: code-coverage
path: artifacts/coverage
- name: Set up Python 3.10
- name: Set up Python 3.9
uses: actions/setup-python@v4
with:
python-version: "3.10"
python-version: "3.9"
- name: Install Nox
run: |
@ -414,7 +455,7 @@ jobs:
- name: Set Exit Status
if: always()
run: |
mkdir exitstatus
python3 -c "import os; os.makedirs('exitstatus', exist_ok=True)"
echo "${{ job.status }}" > exitstatus/${{ github.job }}-${{ inputs.distro-slug }}-${{ env.NOX_SESSION }}-report
- name: Upload Exit Status

View file

@ -69,14 +69,17 @@ if COVERAGE_FILE is None:
IS_DARWIN = sys.platform.lower().startswith("darwin")
IS_WINDOWS = sys.platform.lower().startswith("win")
IS_FREEBSD = sys.platform.lower().startswith("freebsd")
ONEDIR_ARTIFACT_PATH = ARTIFACTS_DIR / "salt"
if IS_WINDOWS:
ONEDIR_PYTHON_PATH = ONEDIR_ARTIFACT_PATH / "Scripts" / "python.exe"
else:
ONEDIR_PYTHON_PATH = ONEDIR_ARTIFACT_PATH / "bin" / "python3"
# Python versions to run against
_PYTHON_VERSIONS = ("3", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10")
# Nox options
# Reuse existing virtualenvs
nox.options.reuse_existing_virtualenvs = True
# Don't fail on missing interpreters
nox.options.error_on_missing_interpreters = False
# Change current directory to REPO_ROOT
os.chdir(str(REPO_ROOT))
@ -117,8 +120,11 @@ def session_run_always(session, *command, **kwargs):
session._runner.global_config.install_only = old_install_only_value
def find_session_runner(session, name, python_version, **kwargs):
name += "-{}".format(python_version)
def find_session_runner(session, name, python_version, onedir=False, **kwargs):
if onedir:
name += "-onedir-{}".format(ONEDIR_PYTHON_PATH)
else:
name += "-{}".format(python_version)
for s, _ in session._runner.manifest.list_all_sessions():
if name not in s.signatures:
continue
@ -301,11 +307,18 @@ def _upgrade_pip_setuptools_and_wheel(session, upgrade=True):
def _install_requirements(
session, transport, *extra_requirements, requirements_type="ci"
session,
transport,
*extra_requirements,
requirements_type="ci",
onedir=False,
):
if not _upgrade_pip_setuptools_and_wheel(session):
return False
if onedir and not IS_WINDOWS and not IS_DARWIN and not IS_FREEBSD:
session_run_always(session, "python3", "-m", "relenv", "toolchain", "fetch")
# Install requirements
requirements_file = _get_pip_requirements_file(
session, transport, requirements_type=requirements_type
@ -1022,9 +1035,12 @@ def _pytest(session, coverage, cmd_args, env=None):
session.run("python", "-m", "pytest", *args, env=env)
def _ci_test(session, transport):
def _ci_test(session, transport, onedir=False):
# Install requirements
_install_requirements(session, transport)
_install_requirements(session, transport, onedir=onedir)
env = {
"ONEDIR_TESTRUN": "1",
}
chunks = {
"unit": [
"tests/unit",
@ -1086,7 +1102,7 @@ def _ci_test(session, transport):
]
+ chunk_cmd
)
_pytest(session, track_code_coverage, pytest_args)
_pytest(session, track_code_coverage, pytest_args, env=env)
except CommandFailed:
if rerun_failures is False:
raise
@ -1106,7 +1122,7 @@ def _ci_test(session, transport):
]
+ chunk_cmd
)
_pytest(session, track_code_coverage, pytest_args)
_pytest(session, track_code_coverage, pytest_args, env=env)
@nox.session(python=_PYTHON_VERSIONS, name="ci-test")
@ -1119,6 +1135,38 @@ def ci_test_tcp(session):
_ci_test(session, "tcp")
@nox.session(
python=str(ONEDIR_PYTHON_PATH),
name="ci-test-onedir",
venv_params=["--system-site-packages"],
)
def ci_test_onedir(session):
if not ONEDIR_ARTIFACT_PATH.exists():
session.error(
"The salt onedir artifact, expected to be in '{}', was not found".format(
ONEDIR_ARTIFACT_PATH.relative_to(REPO_ROOT)
)
)
_ci_test(session, "zeromq", onedir=True)
@nox.session(
python=str(ONEDIR_PYTHON_PATH),
name="ci-test-onedir-tcp",
venv_params=["--system-site-packages"],
)
def ci_test_onedir_tcp(session):
if not ONEDIR_ARTIFACT_PATH.exists():
session.error(
"The salt onedir artifact, expected to be in '{}', was not found".format(
ONEDIR_ARTIFACT_PATH.relative_to(REPO_ROOT)
)
)
_ci_test(session, "tcp", onedir=True)
@nox.session(python="3", name="report-coverage")
def report_coverage(session):
_report_coverage(session)

View file

@ -976,11 +976,17 @@ class VM:
"--exclude",
".pytest_cache/",
"--exclude",
"artifacts/",
"--exclude",
f"{STATE_DIR.relative_to(REPO_ROOT)}{os.path.sep}",
"--exclude",
"*.py~",
# We need to include artifacts/ to be able to include artifacts/salt
"--include",
"artifacts/",
"--include",
"artifacts/salt",
# But we also want to exclude all other entries under artifacts/
"--exclude",
"artifacts/*",
]
if self.is_windows:
# Symlinks aren't handled properly on windows, just replace the