saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-16 09:40:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Build CI dependencies on separate jobs

Browse source 
Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>
This commit is contained in:
Pedro Algarvio 2023-09-28 11:01:47 +01:00 • committed by Pedro Algarvio
parent 4d4ae4d3ce
commit e7612b23d7
26 changed files with 3431 additions and 820 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/workflows/build-deb-packages.yml vendored
View file

@ -1,5 +1,5 @@
---
name: Build Debian Packages
name: Build DEB Packages
on:
workflow_call:
@ -34,6 +34,8 @@ jobs:
arch:
- x86_64
- aarch64
source:
- ${{ inputs.source }}
container:
image: ghcr.io/saltstack/salt-ci-containers/packaging:debian-12

128
.github/workflows/build-deps-ci-action-macos.yml vendored Normal file
View file

@ -0,0 +1,128 @@
---
name: Test Dependencies
on:
workflow_call:
inputs:
distro-slug:
required: true
type: string
description: The OS slug to run tests against
nox-session:
required: true
type: string
description: The nox session to run
salt-version:
type: string
required: true
description: The Salt version to set prior to running tests.
cache-prefix:
required: true
type: string
description: Seed used to invalidate caches
platform:
required: true
type: string
description: The platform being tested
arch:
required: true
type: string
description: The platform arch being tested
nox-version:
required: true
type: string
description: The nox version to install
python-version:
required: false
type: string
description: The python version to run tests with
default: "3.10"
package-name:
required: false
type: string
description: The onedir package name to use
default: salt
env:
COLUMNS: 190
PIP_INDEX_URL: "https://pypi-proxy.saltstack.net/root/local/+simple/"
PIP_EXTRA_INDEX_URL: "https://pypi.org/simple"
jobs:
dependencies:
name: Test Dependencies
runs-on: ${{ inputs.distro-slug }}
timeout-minutes: 90
steps:
- name: Checkout Source Code
uses: actions/checkout@v3
- name: Cache nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
id: nox-dependencies-cache
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{
hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py')
}}
- name: Download Onedir Tarball as an Artifact
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
path: artifacts/
- name: Decompress Onedir Tarball
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
- name: Set up Python ${{ inputs.python-version }}
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: actions/setup-python@v4
with:
python-version: "${{ inputs.python-version }}"
- name: Install System Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
brew install openssl@3
- name: Install Nox
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
python3 -m pip install 'nox==${{ inputs.nox-version }}'
- name: Install Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
env:
PRINT_TEST_SELECTION: "0"
PRINT_SYSTEM_INFO: "0"
run: |
export PYCURL_SSL_LIBRARY=openssl
export LDFLAGS="-L/usr/local/opt/openssl@3/lib"
export CPPFLAGS="-I/usr/local/opt/openssl@3/include"
export PKG_CONFIG_PATH="/usr/local/opt/openssl@3/lib/pkgconfig"
nox --install-only -e ${{ inputs.nox-session }}
- name: Cleanup .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
nox -e "pre-archive-cleanup(pkg=False)"
- name: Compress .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
nox -e compress-dependencies -- ${{ inputs.distro-slug }}
- name: Upload Nox Requirements Tarball
uses: actions/upload-artifact@v3
with:
name: nox-${{ inputs.distro-slug }}-${{ inputs.nox-session }}
path: nox.${{ inputs.distro-slug }}.tar.*

152
.github/workflows/build-deps-ci-action.yml vendored Normal file
View file

@ -0,0 +1,152 @@
---
name: Test Dependencies
on:
workflow_call:
inputs:
distro-slug:
required: true
type: string
description: The OS slug to run tests against
nox-session:
required: true
type: string
description: The nox session to run
salt-version:
type: string
required: true
description: The Salt version to set prior to running tests.
cache-prefix:
required: true
type: string
description: Seed used to invalidate caches
platform:
required: true
type: string
description: The platform being tested
arch:
required: true
type: string
description: The platform arch being tested
nox-version:
required: true
type: string
description: The nox version to install
python-version:
required: false
type: string
description: The python version to run tests with
default: "3.10"
package-name:
required: false
type: string
description: The onedir package name to use
default: salt
env:
COLUMNS: 190
AWS_MAX_ATTEMPTS: "10"
AWS_RETRY_MODE: "adaptive"
PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
PIP_EXTRA_INDEX_URL: https://pypi.org/simple
jobs:
dependencies:
name: Test Dependencies
runs-on:
- self-hosted
- linux
- bastion
timeout-minutes: 90
steps:
- name: Checkout Source Code
uses: actions/checkout@v3
- name: Cache nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
id: nox-dependencies-cache
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{
hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py')
}}
- name: Download Onedir Tarball as an Artifact
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
path: artifacts/
- name: Decompress Onedir Tarball
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
- name: PyPi Proxy
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
sed -i '7s;^;--index-url=https://pypi-proxy.saltstack.net/root/local/+simple/ --extra-index-url=https://pypi.org/simple\n;' requirements/static/ci/*/*.txt
- name: Setup Python Tools Scripts
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-python-tools-scripts
- name: Get Salt Project GitHub Actions Bot Environment
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"
- name: Start VM
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
id: spin-up-vm
run: |
tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ inputs.distro-slug }}
- name: List Free Space
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm ssh ${{ inputs.distro-slug }} -- df -h || true
- name: Upload Checkout To VM
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm rsync ${{ inputs.distro-slug }}
- name: Install Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm install-dependencies --nox-session=${{ inputs.nox-session }} ${{ inputs.distro-slug }}
- name: Cleanup .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm pre-archive-cleanup ${{ inputs.distro-slug }}
- name: Compress .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm compress-dependencies ${{ inputs.distro-slug }}
- name: Download Compressed .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm download-dependencies ${{ inputs.distro-slug }}
- name: Destroy VM
if: always() && steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm destroy --no-wait ${{ inputs.distro-slug }}
- name: Upload Nox Requirements Tarball
uses: actions/upload-artifact@v3
with:
name: nox-${{ inputs.distro-slug }}-${{ inputs.nox-session }}
path: nox.${{ inputs.distro-slug }}.tar.*

2
.github/workflows/build-macos-packages.yml vendored
View file

@ -39,6 +39,8 @@ jobs:
matrix:
arch:
- x86_64
source:
- ${{ inputs.source }}
runs-on:
- macos-12

2
.github/workflows/build-rpm-packages.yml vendored
View file

@ -37,6 +37,8 @@ jobs:
arch:
- x86_64
- aarch64
source:
- ${{ inputs.source }}
container:
image: ghcr.io/saltstack/salt-ci-containers/packaging:centosstream-9

2
.github/workflows/build-windows-packages.yml vendored
View file

@ -41,6 +41,8 @@ jobs:
arch:
- x86
- amd64
source:
- ${{ inputs.source }}
runs-on:
- windows-latest

784
.github/workflows/ci.yml vendored
View file

File diff suppressed because it is too large Load diff

784
.github/workflows/nightly.yml vendored
View file

File diff suppressed because it is too large Load diff

33
.github/workflows/release.yml vendored
View file

@ -240,6 +240,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -262,6 +263,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -284,6 +286,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -306,6 +309,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -328,6 +332,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -350,6 +355,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -372,6 +378,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -394,6 +401,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -416,6 +424,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -438,6 +447,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -460,6 +470,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -482,6 +493,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -504,6 +516,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -526,6 +539,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -548,6 +562,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -570,6 +585,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -592,6 +608,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -614,6 +631,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -636,6 +654,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -658,6 +677,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -680,6 +700,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -702,6 +723,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -724,6 +746,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -746,6 +769,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -768,6 +792,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -790,6 +815,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
@ -812,6 +838,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: onedir
@ -834,6 +861,7 @@ jobs:
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: release
nox-version: 2022.8.7
python-version: "3.10"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: onedir
@ -857,6 +885,7 @@ jobs:
environment: release
skip-code-coverage: true
nox-version: 2022.8.7
python-version: "3.10"
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: package
secrets: inherit
@ -879,6 +908,7 @@ jobs:
environment: release
skip-code-coverage: true
nox-version: 2022.8.7
python-version: "3.10"
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: onedir
secrets: inherit
@ -902,6 +932,7 @@ jobs:
environment: release
skip-code-coverage: true
nox-version: 2022.8.7
python-version: "3.10"
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
secrets: inherit
@ -924,6 +955,7 @@ jobs:
environment: release
skip-code-coverage: true
nox-version: 2022.8.7
python-version: "3.10"
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
secrets: inherit
@ -946,6 +978,7 @@ jobs:
environment: release
skip-code-coverage: true
nox-version: 2022.8.7
python-version: "3.10"
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
secrets: inherit

784
.github/workflows/scheduled.yml vendored
View file

File diff suppressed because it is too large Load diff

785
.github/workflows/staging.yml vendored
View file

File diff suppressed because it is too large Load diff

95
.github/workflows/templates/build-ci-deps.yml.jinja vendored Normal file
View file

@ -0,0 +1,95 @@
<%- for slug, display_name, arch in (("windows-2016", "Windows 2016", "amd64"),
("windows-2019", "Windows 2019", "amd64"),
("windows-2022", "Windows 2022", "amd64")) %>
<{ slug.replace(".", "") }>-ci-deps:
<%- do test_salt_needs.append(slug.replace(".", "") + "-ci-deps") %>
name: <{ display_name }>
if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }}
needs:
- prepare-workflow
- build-salt-onedir
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: <{ slug }>
nox-session: ci-test-onedir
platform: windows
arch: amd64
nox-version: <{ nox_version }>
python-version: "<{ gh_actions_workflows_python_version }>"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }>
<%- endfor %>
<%- for slug, display_name, arch in (("macos-12", "macOS 12", "x86_64"),) %>
<{ slug.replace(".", "") }>-ci-deps:
<%- do test_salt_needs.append(slug.replace(".", "") + "-ci-deps") %>
name: <{ display_name }>
if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }}
needs:
- prepare-workflow
- build-salt-onedir
uses: ./.github/workflows/build-deps-ci-action-macos.yml
with:
distro-slug: <{ slug }>
nox-session: ci-test-onedir
platform: darwin
arch: x86_64
nox-version: <{ nox_version }>
python-version: "<{ gh_actions_workflows_python_version }>"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }>
<%- endfor %>
<%- for slug, display_name, arch in (("almalinux-8", "Alma Linux 8", "x86_64"),
("almalinux-8-arm64", "Alma Linux 8 Arm64", "aarch64"),
("almalinux-9", "Alma Linux 9", "x86_64"),
("almalinux-9-arm64", "Alma Linux 9 Arm64", "aarch64"),
("amazonlinux-2", "Amazon Linux 2", "x86_64"),
("amazonlinux-2-arm64", "Amazon Linux 2 Arm64", "aarch64"),
("archlinux-lts", "Arch Linux LTS", "x86_64"),
("centos-7", "CentOS 7", "x86_64"),
("centos-7-arm64", "CentOS 7 Arm64", "aarch64"),
("centosstream-8", "CentOS Stream 8", "x86_64"),
("centosstream-8-arm64", "CentOS Stream 8 Arm64", "aarch64"),
("centosstream-9", "CentOS Stream 9", "x86_64"),
("centosstream-9-arm64", "CentOS Stream 9 Arm64", "aarch64"),
("debian-10", "Debian 10", "x86_64"),
("debian-11", "Debian 11", "x86_64"),
("debian-11-arm64", "Debian 11 Arm64", "aarch64"),
("fedora-37", "Fedora 37", "x86_64"),
("fedora-37-arm64", "Fedora 37 Arm64", "aarch64"),
("fedora-38", "Fedora 38", "x86_64"),
("fedora-38-arm64", "Fedora 38 Arm64", "aarch64"),
("opensuse-15", "Opensuse 15", "x86_64"),
("photonos-3", "Photon OS 3", "x86_64"),
("photonos-4", "Photon OS 4", "x86_64"),
("photonos-4-arm64", "Photon OS 4 Arm64", "aarch64"),
("ubuntu-20.04", "Ubuntu 20.04", "x86_64"),
("ubuntu-20.04-arm64", "Ubuntu 20.04 Arm64", "aarch64"),
("ubuntu-22.04", "Ubuntu 22.04", "x86_64"),
("ubuntu-22.04-arm64", "Ubuntu 22.04 Arm64", "aarch64")) %>
<{ slug.replace(".", "") }>-ci-deps:
<%- do test_salt_needs.append(slug.replace(".", "") + "-ci-deps") %>
name: <{ display_name }>
if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }}
needs:
- prepare-workflow
- build-salt-onedir
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: <{ slug }>
nox-session: ci-test-onedir
platform: linux
arch: <{ arch }>
nox-version: <{ nox_version }>
python-version: "<{ gh_actions_workflows_python_version }>"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }>
<%- endfor %>

5
.github/workflows/templates/build-packages.yml.jinja vendored
View file

@ -7,9 +7,12 @@
<%- for backend in ("onedir", "src") %>
<%- set job_name = "build-{}-pkgs-{}".format(pkg_type, backend) %>
<%- if backend == "src" %>
<%- do conclusion_needs.append(job_name) %>
<%- endif %>
<{ job_name }>:
name: Build <{ display_name }> Packages (<{ backend }>)
name: Build Packages
if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['<{ runner_type }>'] }}
needs:
- prepare-workflow

11
.github/workflows/templates/ci.yml.jinja vendored
View file

@ -301,15 +301,18 @@
<%- include "build-packages.yml.jinja" %>
<%- endif %>
<%- set pkg_tests_job_name = "pkg-tests" %>
<%- set salt_tests_job_name = "salt-tests" %>
<%- if includes.get(pkg_tests_job_name, True) or includes.get(salt_tests_job_name, True) %>
<%- include "build-ci-deps.yml.jinja" %>
<%- endif %>
<%- set job_name = "pkg-tests" %>
<%- if includes.get(job_name, True) %>
<%- if includes.get(pkg_tests_job_name, True) %>
<%- include "test-salt-pkg.yml.jinja" %>
<%- endif %>
<%- set job_name = "salt-tests" %>
<%- if includes.get(job_name, True) %>
<%- if includes.get(salt_tests_job_name, True) %>
<%- include "test-salt.yml.jinja" %>
<%- endif %>

1
.github/workflows/templates/layout.yml.jinja vendored
View file

@ -9,6 +9,7 @@
<%- set skip_junit_reports_check = skip_junit_reports_check|default("${{ github.event_name == 'pull_request' }}") %>
<%- set gpg_key_id = "64CBBC8173D76B3F" %>
<%- set prepare_actual_release = prepare_actual_release | default(False) %>
<%- set gh_actions_workflows_python_version = "3.10" %>
---
<%- block name %>
name: <{ workflow_name }>

3
.github/workflows/templates/test-pkg-repo-downloads.yml.jinja vendored
View file

@ -58,6 +58,7 @@
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
environment: <{ gh_environment }>
nox-version: <{ nox_version }>
python-version: "<{ gh_actions_workflows_python_version }>"
skip-code-coverage: true
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: <{ pkg_type.lower() }>
@ -97,6 +98,7 @@
environment: <{ gh_environment }>
skip-code-coverage: true
nox-version: <{ nox_version }>
python-version: "<{ gh_actions_workflows_python_version }>"
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
pkg-type: <{ pkg_type.lower() }>
secrets: inherit
@ -135,6 +137,7 @@
environment: <{ gh_environment }>
skip-code-coverage: true
nox-version: <{ nox_version }>
python-version: "<{ gh_actions_workflows_python_version }>"
latest-release: "${{ needs.prepare-workflow.outputs.latest-release }}"
secrets: inherit
<%- endfor %>

12
.github/workflows/templates/test-salt-pkg.yml.jinja vendored
View file

@ -39,11 +39,12 @@
<{ job_name }>:
<%- do test_salt_pkg_needs.append(job_name) %>
name: <{ display_name }> Package Tests
name: <{ display_name }>
if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test-pkg'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }}
needs:
- prepare-workflow
- build-<{ pkg_type }>-pkgs-onedir
- <{ slug.replace(".", "") }>-ci-deps
uses: ./.github/workflows/test-packages-action.yml
with:
distro-slug: <{ slug }>
@ -53,6 +54,7 @@
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
pkg-type: <{ pkg_type }>
nox-version: <{ nox_version }>
python-version: "<{ gh_actions_workflows_python_version }>"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }>
skip-code-coverage: <{ skip_test_coverage_check }>
skip-junit-reports: <{ skip_junit_reports_check }>
@ -67,11 +69,12 @@
<{ job_name }>:
<%- do test_salt_pkg_needs.append(job_name) %>
name: <{ display_name }> Package Tests
name: <{ display_name }>
if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test-pkg'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }}
needs:
- prepare-workflow
- build-macos-pkgs-onedir
- <{ slug.replace(".", "") }>-ci-deps
uses: ./.github/workflows/test-packages-action-macos.yml
with:
distro-slug: <{ slug }>
@ -81,6 +84,7 @@
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
pkg-type: macos
nox-version: <{ nox_version }>
python-version: "<{ gh_actions_workflows_python_version }>"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }>
skip-code-coverage: <{ skip_test_coverage_check }>
skip-junit-reports: <{ skip_junit_reports_check }>
@ -97,11 +101,12 @@
<{ job_name }>:
<%- do test_salt_pkg_needs.append(job_name) %>
name: <{ display_name }> <{ pkg_type }> Package Tests
name: <{ display_name }>
if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test-pkg'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }}
needs:
- prepare-workflow
- build-windows-pkgs-onedir
- <{ slug.replace(".", "") }>-ci-deps
uses: ./.github/workflows/test-packages-action.yml
with:
distro-slug: <{ slug }>
@ -111,6 +116,7 @@
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
pkg-type: <{ pkg_type }>
nox-version: <{ nox_version }>
python-version: "<{ gh_actions_workflows_python_version }>"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }>
skip-code-coverage: <{ skip_test_coverage_check }>
skip-junit-reports: <{ skip_junit_reports_check }>

9
.github/workflows/templates/test-salt.yml.jinja vendored
View file

@ -9,7 +9,7 @@
if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }}
needs:
- prepare-workflow
- build-salt-onedir
- <{ slug.replace(".", "") }>-ci-deps
uses: ./.github/workflows/test-action.yml
with:
distro-slug: <{ slug }>
@ -17,6 +17,7 @@
platform: windows
arch: amd64
nox-version: <{ nox_version }>
python-version: "<{ gh_actions_workflows_python_version }>"
testrun: ${{ needs.prepare-workflow.outputs.testrun }}
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }>
@ -33,7 +34,7 @@
if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }}
needs:
- prepare-workflow
- build-salt-onedir
- <{ slug.replace(".", "") }>-ci-deps
uses: ./.github/workflows/test-action-macos.yml
with:
distro-slug: <{ slug }>
@ -41,6 +42,7 @@
platform: darwin
arch: x86_64
nox-version: <{ nox_version }>
python-version: "<{ gh_actions_workflows_python_version }>"
testrun: ${{ needs.prepare-workflow.outputs.testrun }}
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }>
@ -76,7 +78,7 @@
if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }}
needs:
- prepare-workflow
- build-salt-onedir
- <{ slug.replace(".", "") }>-ci-deps
uses: ./.github/workflows/test-action.yml
with:
distro-slug: <{ slug }>
@ -84,6 +86,7 @@
platform: linux
arch: <{ arch }>
nox-version: <{ nox_version }>
python-version: "<{ gh_actions_workflows_python_version }>"
testrun: ${{ needs.prepare-workflow.outputs.testrun }}
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|<{ python_version }>

83
.github/workflows/test-action-macos.yml vendored
View file

@ -66,7 +66,7 @@ env:
jobs:
generate-matrix:
name: Generate Test Matrix
name: Test Matrix
runs-on: ${{ github.event.repository.private && fromJSON('["self-hosted", "linux", "x86_64"]') || 'ubuntu-latest' }}
outputs:
matrix-include: ${{ steps.generate-matrix.outputs.matrix }}
@ -84,87 +84,12 @@ jobs:
echo "$TEST_MATRIX"
echo "matrix=$TEST_MATRIX" >> "$GITHUB_OUTPUT"
dependencies:
name: Setup Test Dependencies
runs-on: ${{ inputs.distro-slug }}
timeout-minutes: 90
steps:
- name: Checkout Source Code
uses: actions/checkout@v3
- name: Cache nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
id: nox-dependencies-cache
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ hashFiles('requirements/**/*.txt', 'noxfile.py') }}
- name: Download Onedir Tarball as an Artifact
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
path: artifacts/
- name: Decompress Onedir Tarball
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
- name: Set up Python ${{ inputs.python-version }}
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: actions/setup-python@v4
with:
python-version: "${{ inputs.python-version }}"
- name: Install System Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
brew install openssl@3
- name: Install Nox
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
python3 -m pip install 'nox==${{ inputs.nox-version }}'
- name: Install Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
env:
PRINT_TEST_SELECTION: "0"
PRINT_SYSTEM_INFO: "0"
run: |
export PYCURL_SSL_LIBRARY=openssl
export LDFLAGS="-L/usr/local/opt/openssl@3/lib"
export CPPFLAGS="-I/usr/local/opt/openssl@3/include"
export PKG_CONFIG_PATH="/usr/local/opt/openssl@3/lib/pkgconfig"
nox --install-only -e ${{ inputs.nox-session }}
- name: Cleanup .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
nox -e "pre-archive-cleanup(pkg=False)"
- name: Compress .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
nox -e compress-dependencies -- ${{ inputs.distro-slug }}
- name: Upload Nox Requirements Tarball
uses: actions/upload-artifact@v3
with:
name: nox-${{ inputs.distro-slug }}-${{ inputs.nox-session }}
path: nox.${{ inputs.distro-slug }}.tar.*
test:
name: Test
runs-on: ${{ inputs.distro-slug }}
timeout-minutes: 360 # 6 Hours
needs:
- generate-matrix
- dependencies
strategy:
fail-fast: false
matrix:
@ -201,7 +126,9 @@ jobs:
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ hashFiles('requirements/**/*.txt', 'noxfile.py') }}
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{
hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py')
}}
# If we get a cache miss here it means the dependencies step failed to save the cache
fail-on-cache-miss: true
@ -445,7 +372,7 @@ jobs:
report:
name: Reports for ${{ inputs.distro-slug }}
name: Test Reports
if: always() && inputs.skip-code-coverage == false && needs.test.result != 'cancelled' && needs.test.result != 'skipped'
runs-on: ${{ github.event.repository.private && fromJSON('["self-hosted", "linux", "x86_64"]') || 'ubuntu-latest' }}
needs:

110
.github/workflows/test-action.yml vendored
View file

@ -36,6 +36,11 @@ on:
required: true
type: string
description: The nox version to install
python-version:
required: false
type: string
description: The python version to run tests with
default: "3.10"
package-name:
required: false
type: string
@ -63,7 +68,7 @@ env:
jobs:
generate-matrix:
name: Generate Test Matrix
name: Test Matrix
runs-on:
- self-hosted
- linux
@ -84,102 +89,6 @@ jobs:
echo "$TEST_MATRIX"
echo "matrix=$TEST_MATRIX" >> "$GITHUB_OUTPUT"
dependencies:
name: Setup Test Dependencies
runs-on:
- self-hosted
- linux
- bastion
timeout-minutes: 90
steps:
- name: Checkout Source Code
uses: actions/checkout@v3
- name: Cache nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
id: nox-dependencies-cache
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}
- name: Download Onedir Tarball as an Artifact
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
path: artifacts/
- name: Decompress Onedir Tarball
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
- name: PyPi Proxy
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
sed -i '7s;^;--index-url=https://pypi-proxy.saltstack.net/root/local/+simple/ --extra-index-url=https://pypi.org/simple\n;' requirements/static/ci/*/*.txt
- name: Setup Python Tools Scripts
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-python-tools-scripts
- name: Get Salt Project GitHub Actions Bot Environment
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"
- name: Start VM
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
id: spin-up-vm
run: |
tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ inputs.distro-slug }}
- name: List Free Space
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm ssh ${{ inputs.distro-slug }} -- df -h || true
- name: Upload Checkout To VM
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm rsync ${{ inputs.distro-slug }}
- name: Install Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm install-dependencies --nox-session=${{ inputs.nox-session }} ${{ inputs.distro-slug }}
- name: Cleanup .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm pre-archive-cleanup ${{ inputs.distro-slug }}
- name: Compress .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm compress-dependencies ${{ inputs.distro-slug }}
- name: Download Compressed .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm download-dependencies ${{ inputs.distro-slug }}
- name: Destroy VM
if: always() && steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm destroy --no-wait ${{ inputs.distro-slug }}
- name: Upload Nox Requirements Tarball
uses: actions/upload-artifact@v3
with:
name: nox-${{ inputs.distro-slug }}-${{ inputs.nox-session }}
path: nox.${{ inputs.distro-slug }}.tar.*
test:
name: Test
runs-on:
@ -188,7 +97,6 @@ jobs:
- bastion
timeout-minutes: 300 # 5 Hours - More than this and something is wrong
needs:
- dependencies
- generate-matrix
strategy:
fail-fast: false
@ -222,7 +130,9 @@ jobs:
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{
hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py')
}}
# If we get a cache miss here it means the dependencies step failed to save the cache
fail-on-cache-miss: true
@ -409,7 +319,7 @@ jobs:
report:
name: Reports for ${{ inputs.distro-slug }}
name: Test Reports
if: always() && inputs.skip-code-coverage == false && needs.test.result != 'cancelled' && needs.test.result != 'skipped'
runs-on:
- self-hosted

99
.github/workflows/test-package-downloads-action-linux.yml vendored
View file

@ -39,6 +39,11 @@ on:
required: true
type: string
description: The nox version to install
python-version:
required: false
type: string
description: The python version to run tests with
default: "3.10"
package-name:
required: false
type: string
@ -65,100 +70,14 @@ env:
jobs:
dependencies:
name: Setup Test Dependencies
runs-on:
- self-hosted
- linux
- bastion
timeout-minutes: 90
steps:
- name: Checkout Source Code
uses: actions/checkout@v3
- name: Cache nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
id: nox-dependencies-cache
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}
- name: Download Onedir Tarball as an Artifact
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
path: artifacts/
- name: Decompress Onedir Tarball
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
- name: Setup Python Tools Scripts
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-python-tools-scripts
- name: Get Salt Project GitHub Actions Bot Environment
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"
- name: Start VM
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
id: spin-up-vm
run: |
tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ inputs.distro-slug }}
- name: List Free Space
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm ssh ${{ inputs.distro-slug }} -- df -h || true
- name: Upload Checkout To VM
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm rsync ${{ inputs.distro-slug }}
- name: Install Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm install-dependencies --nox-session=${{ inputs.nox-session }} ${{ inputs.distro-slug }}
- name: Cleanup .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm pre-archive-cleanup ${{ inputs.distro-slug }}
- name: Compress .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm compress-dependencies ${{ inputs.distro-slug }}
- name: Download Compressed .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm download-dependencies ${{ inputs.distro-slug }}
- name: Destroy VM
if: always() && steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm destroy --no-wait ${{ inputs.distro-slug }}
test:
name: Test
name: Test Pkg Download
runs-on:
- self-hosted
- linux
- bastion
environment: ${{ inputs.environment }}
timeout-minutes: 120 # 2 Hours - More than this and something is wrong
needs:
- dependencies
steps:
- name: Checkout Source Code
@ -181,7 +100,9 @@ jobs:
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{
hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py')
}}
# If we get a cache miss here it means the dependencies step failed to save the cache
fail-on-cache-miss: true
@ -287,7 +208,7 @@ jobs:
!artifacts/salt-*.tar.*
report:
name: Reports for ${{ inputs.distro-slug }}(${{ inputs.arch }})
name: Test Pkg Download Reports
runs-on:
- self-hosted
- linux

78
.github/workflows/test-package-downloads-action-macos.yml vendored
View file

@ -70,81 +70,11 @@ env:
jobs:
dependencies:
name: Setup Test Dependencies
runs-on: ${{ inputs.distro-slug }}
timeout-minutes: 90
steps:
- name: Checkout Source Code
uses: actions/checkout@v3
- name: Cache nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
id: nox-dependencies-cache
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}
- name: Download Onedir Tarball as an Artifact
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
path: artifacts/
- name: Decompress Onedir Tarball
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
- name: Set up Python ${{ inputs.python-version }}
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: actions/setup-python@v4
with:
python-version: "${{ inputs.python-version }}"
update-environment: true
- name: Install System Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
brew install openssl@3
- name: Install Nox
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
python3 -m pip install 'nox==${{ inputs.nox-version }}'
- name: Install Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
env:
PRINT_TEST_SELECTION: "0"
PRINT_SYSTEM_INFO: "0"
run: |
export PYCURL_SSL_LIBRARY=openssl
export LDFLAGS="-L/usr/local/opt/openssl@3/lib"
export CPPFLAGS="-I/usr/local/opt/openssl@3/include"
export PKG_CONFIG_PATH="/usr/local/opt/openssl@3/lib/pkgconfig"
nox --force-color --install-only -e ${{ inputs.nox-session }}
- name: Cleanup .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
nox --force-color -e "pre-archive-cleanup(pkg=False)"
- name: Compress .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
nox --force-color -e compress-dependencies -- ${{ inputs.distro-slug }}
test:
name: Test
name: Test Pkg Download
runs-on: ${{ inputs.distro-slug }}
environment: ${{ inputs.environment }}
timeout-minutes: 120 # 2 Hours - More than this and something is wrong
needs:
- dependencies
steps:
- name: Checkout Source Code
@ -181,7 +111,9 @@ jobs:
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{
hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py')
}}
# If we get a cache miss here it means the dependencies step failed to save the cache
fail-on-cache-miss: true
@ -261,7 +193,7 @@ jobs:
!artifacts/salt-*.tar.*
report:
name: Reports for ${{ inputs.distro-slug }}(${{ inputs.arch }})
name: Test Pkg Download Reports
runs-on: ${{ github.event.repository.private && fromJSON('["self-hosted", "linux", "x86_64"]') || 'ubuntu-latest' }}
environment: ${{ inputs.environment }}
if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped'

99
.github/workflows/test-package-downloads-action-windows.yml vendored
View file

@ -39,6 +39,11 @@ on:
required: true
type: string
description: The nox version to install
python-version:
required: false
type: string
description: The python version to run tests with
default: "3.10"
package-name:
required: false
type: string
@ -70,100 +75,14 @@ env:
jobs:
dependencies:
name: Setup Test Dependencies
runs-on:
- self-hosted
- linux
- bastion
timeout-minutes: 90
steps:
- name: Checkout Source Code
uses: actions/checkout@v3
- name: Cache nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
id: nox-dependencies-cache
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}
- name: Download Onedir Tarball as an Artifact
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
path: artifacts/
- name: Decompress Onedir Tarball
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
- name: Setup Python Tools Scripts
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-python-tools-scripts
- name: Get Salt Project GitHub Actions Bot Environment
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"
- name: Start VM
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
id: spin-up-vm
run: |
tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ inputs.distro-slug }}
- name: List Free Space
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm ssh ${{ inputs.distro-slug }} -- df -h || true
- name: Upload Checkout To VM
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm rsync ${{ inputs.distro-slug }}
- name: Install Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm install-dependencies --nox-session=${{ inputs.nox-session }} ${{ inputs.distro-slug }}
- name: Cleanup .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm pre-archive-cleanup ${{ inputs.distro-slug }}
- name: Compress .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm compress-dependencies ${{ inputs.distro-slug }}
- name: Download Compressed .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm download-dependencies ${{ inputs.distro-slug }}
- name: Destroy VM
if: always() && steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm destroy --no-wait ${{ inputs.distro-slug }}
test:
name: Test
name: Test Pkg Download
runs-on:
- self-hosted
- linux
- bastion
environment: ${{ inputs.environment }}
timeout-minutes: 120 # 2 Hours - More than this and something is wrong
needs:
- dependencies
steps:
- name: Checkout Source Code
@ -186,7 +105,9 @@ jobs:
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|test-pkg-download-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{
hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py')
}}
# If we get a cache miss here it means the dependencies step failed to save the cache
fail-on-cache-miss: true
@ -292,7 +213,7 @@ jobs:
!artifacts/salt-*.tar.*
report:
name: Reports for ${{ inputs.distro-slug }}(${{ inputs.arch }})
name: Test Pkg Download Reports
runs-on:
- self-hosted
- linux

81
.github/workflows/test-packages-action-macos.yml vendored
View file

@ -72,7 +72,7 @@ env:
jobs:
generate-matrix:
name: Generate Package Test Matrix
name: Test Pkg Matrix
runs-on: ${{ github.event.repository.private && fromJSON('["self-hosted", "linux", "x86_64"]') || 'ubuntu-latest' }}
outputs:
pkg-matrix-include: ${{ steps.generate-pkg-matrix.outputs.matrix }}
@ -88,83 +88,12 @@ jobs:
run: |
tools ci pkg-matrix ${{ inputs.distro-slug }} ${{ inputs.pkg-type }} --testing-releases ${{ join(fromJSON(inputs.testing-releases), ' ') }}
dependencies:
name: Setup Test Dependencies
needs:
- generate-matrix
runs-on: ${{ inputs.distro-slug }}
timeout-minutes: 90
steps:
- name: Checkout Source Code
uses: actions/checkout@v3
- name: Cache nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
id: nox-dependencies-cache
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}
- name: Download Onedir Tarball as an Artifact
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
path: artifacts/
- name: Decompress Onedir Tarball
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
- name: Set up Python ${{ inputs.python-version }}
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: actions/setup-python@v4
with:
python-version: "${{ inputs.python-version }}"
- name: Install System Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
brew install openssl@3
- name: Install Nox
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
python3 -m pip install 'nox==${{ inputs.nox-version }}'
- name: Install Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
env:
PRINT_TEST_SELECTION: "0"
PRINT_SYSTEM_INFO: "0"
run: |
export PYCURL_SSL_LIBRARY=openssl
export LDFLAGS="-L/usr/local/opt/openssl@3/lib"
export CPPFLAGS="-I/usr/local/opt/openssl@3/include"
export PKG_CONFIG_PATH="/usr/local/opt/openssl@3/lib/pkgconfig"
nox --force-color --install-only -e ${{ inputs.nox-session }}
- name: Cleanup .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
nox --force-color -e "pre-archive-cleanup(pkg=False)"
- name: Compress .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
nox --force-color -e compress-dependencies -- ${{ inputs.distro-slug }}
test:
name: Test
name: Test Pkg
runs-on: ${{ inputs.distro-slug }}
timeout-minutes: 120 # 2 Hours - More than this and something is wrong
needs:
- dependencies
- generate-matrix
strategy:
fail-fast: false
@ -215,7 +144,9 @@ jobs:
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{
hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py')
}}
# If we get a cache miss here it means the dependencies step failed to save the cache
fail-on-cache-miss: true
@ -273,7 +204,7 @@ jobs:
!artifacts/salt-*.tar.*
report:
name: Reports for ${{ inputs.distro-slug }}(${{ matrix.test-chunk }})
name: Test Pkg Reports
runs-on: ${{ github.event.repository.private && fromJSON('["self-hosted", "linux", "x86_64"]') || 'ubuntu-latest' }}
if: always() && (inputs.skip-code-coverage == false || inputs.skip-junit-reports == false) && needs.test.result != 'cancelled' && needs.test.result != 'skipped'
needs:

103
.github/workflows/test-packages-action.yml vendored
View file

@ -35,6 +35,11 @@ on:
required: true
type: string
description: The nox version to install
python-version:
required: false
type: string
description: The python version to run tests with
default: "3.10"
package-name:
required: false
type: string
@ -67,7 +72,7 @@ env:
jobs:
generate-matrix:
name: Generate Package Test Matrix
name: Test ${{ inputs.pkg-type }} Pkg Matrix
runs-on:
- self-hosted
- linux
@ -86,96 +91,9 @@ jobs:
run: |
tools ci pkg-matrix ${{ inputs.distro-slug }} ${{ inputs.pkg-type }} --testing-releases ${{ join(fromJSON(inputs.testing-releases), ' ') }}
dependencies:
name: Setup Test Dependencies
needs:
- generate-matrix
runs-on:
- self-hosted
- linux
- bastion
timeout-minutes: 90
steps:
- name: Checkout Source Code
uses: actions/checkout@v3
- name: Cache nox.${{ inputs.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
id: nox-dependencies-cache
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}
- name: Download Onedir Tarball as an Artifact
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
path: artifacts/
- name: Decompress Onedir Tarball
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
- name: Setup Python Tools Scripts
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-python-tools-scripts
- name: Get Salt Project GitHub Actions Bot Environment
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"
- name: Start VM
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
id: spin-up-vm
run: |
tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ inputs.distro-slug }}
- name: List Free Space
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm ssh ${{ inputs.distro-slug }} -- df -h || true
- name: Upload Checkout To VM
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm rsync ${{ inputs.distro-slug }}
- name: Install Dependencies
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm install-dependencies --nox-session=${{ inputs.nox-session }} ${{ inputs.distro-slug }}
- name: Cleanup .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm pre-archive-cleanup ${{ inputs.distro-slug }}
- name: Compress .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm compress-dependencies ${{ inputs.distro-slug }}
- name: Download Compressed .nox Directory
if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm download-dependencies ${{ inputs.distro-slug }}
- name: Destroy VM
if: always() && steps.nox-dependencies-cache.outputs.cache-hit != 'true'
run: |
tools --timestamps vm destroy --no-wait ${{ inputs.distro-slug }}
test:
name: Test
name: Test Pkg
runs-on:
- self-hosted
- linux
@ -183,7 +101,6 @@ jobs:
timeout-minutes: 120 # 2 Hours - More than this and something is wrong
needs:
- generate-matrix
- dependencies
strategy:
fail-fast: false
matrix:
@ -220,7 +137,9 @@ jobs:
uses: actions/cache@v3
with:
path: nox.${{ inputs.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ inputs.arch }}|${{ inputs.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{
hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py')
}}
# If we get a cache miss here it means the dependencies step failed to save the cache
fail-on-cache-miss: true
@ -288,7 +207,7 @@ jobs:
!artifacts/salt-*.tar.*
report:
name: Reports for ${{ inputs.distro-slug }}(${{ matrix.test-chunk }})
name: Test Pkg Reports
runs-on:
- self-hosted
- linux

2
tools/ci.py
View file

@ -739,6 +739,8 @@ def pkg_matrix(
"version": version,
}
)
if distro_slug.startswith("windows"):
matrix[-1]["pkg-type"] = pkg_type.upper()
ctx.info("Generated matrix:")
ctx.print(matrix, soft_wrap=True)