saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Even the release workflow should be working now

Browse source 
Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>
This commit is contained in:
Pedro Algarvio 2023-10-09 14:40:41 +01:00 • committed by Pedro Algarvio
parent 42be5ce982
commit 2c6f5d88fd
9 changed files with 1572 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

574
.github/workflows/release.yml vendored
View file

@ -112,7 +112,7 @@ jobs:
- name: Set Cache Seed Output
id: set-cache-seed
run: |
echo "cache-seed=${{ env.CACHE_SEED }}" >> "$GITHUB_OUTPUT"
tools ci define-cache-seed ${{ env.CACHE_SEED }}
download-onedir-artifact:
name: Download Staging Onedir Artifact
@ -162,6 +162,518 @@ jobs:
retention-days: 7
if-no-files-found: error
windows-2016-ci-deps:
name: Windows 2016
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: windows-2016
nox-session: ci-test-onedir
platform: windows
arch: amd64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
windows-2019-ci-deps:
name: Windows 2019
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: windows-2019
nox-session: ci-test-onedir
platform: windows
arch: amd64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
windows-2022-ci-deps:
name: Windows 2022
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: windows-2022
nox-session: ci-test-onedir
platform: windows
arch: amd64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
macos-12-ci-deps:
name: macOS 12
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action-macos.yml
with:
distro-slug: macos-12
nox-session: ci-test-onedir
platform: darwin
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
almalinux-8-ci-deps:
name: Alma Linux 8
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: almalinux-8
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
almalinux-8-arm64-ci-deps:
name: Alma Linux 8 Arm64
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: almalinux-8-arm64
nox-session: ci-test-onedir
platform: linux
arch: aarch64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
almalinux-9-ci-deps:
name: Alma Linux 9
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: almalinux-9
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
almalinux-9-arm64-ci-deps:
name: Alma Linux 9 Arm64
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: almalinux-9-arm64
nox-session: ci-test-onedir
platform: linux
arch: aarch64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
amazonlinux-2-ci-deps:
name: Amazon Linux 2
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: amazonlinux-2
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
amazonlinux-2-arm64-ci-deps:
name: Amazon Linux 2 Arm64
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: amazonlinux-2-arm64
nox-session: ci-test-onedir
platform: linux
arch: aarch64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
archlinux-lts-ci-deps:
name: Arch Linux LTS
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: archlinux-lts
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
centos-7-ci-deps:
name: CentOS 7
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: centos-7
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
centos-7-arm64-ci-deps:
name: CentOS 7 Arm64
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: centos-7-arm64
nox-session: ci-test-onedir
platform: linux
arch: aarch64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
centosstream-8-ci-deps:
name: CentOS Stream 8
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: centosstream-8
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
centosstream-8-arm64-ci-deps:
name: CentOS Stream 8 Arm64
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: centosstream-8-arm64
nox-session: ci-test-onedir
platform: linux
arch: aarch64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
centosstream-9-ci-deps:
name: CentOS Stream 9
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: centosstream-9
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
centosstream-9-arm64-ci-deps:
name: CentOS Stream 9 Arm64
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: centosstream-9-arm64
nox-session: ci-test-onedir
platform: linux
arch: aarch64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
debian-10-ci-deps:
name: Debian 10
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: debian-10
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
debian-11-ci-deps:
name: Debian 11
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: debian-11
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
debian-11-arm64-ci-deps:
name: Debian 11 Arm64
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: debian-11-arm64
nox-session: ci-test-onedir
platform: linux
arch: aarch64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
fedora-37-ci-deps:
name: Fedora 37
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: fedora-37
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
fedora-37-arm64-ci-deps:
name: Fedora 37 Arm64
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: fedora-37-arm64
nox-session: ci-test-onedir
platform: linux
arch: aarch64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
fedora-38-ci-deps:
name: Fedora 38
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: fedora-38
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
fedora-38-arm64-ci-deps:
name: Fedora 38 Arm64
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: fedora-38-arm64
nox-session: ci-test-onedir
platform: linux
arch: aarch64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
opensuse-15-ci-deps:
name: Opensuse 15
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: opensuse-15
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
photonos-3-ci-deps:
name: Photon OS 3
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: photonos-3
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
photonos-4-ci-deps:
name: Photon OS 4
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: photonos-4
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
photonos-4-arm64-ci-deps:
name: Photon OS 4 Arm64
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: photonos-4-arm64
nox-session: ci-test-onedir
platform: linux
arch: aarch64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
ubuntu-2004-ci-deps:
name: Ubuntu 20.04
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: ubuntu-20.04
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
ubuntu-2004-arm64-ci-deps:
name: Ubuntu 20.04 Arm64
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: ubuntu-20.04-arm64
nox-session: ci-test-onedir
platform: linux
arch: aarch64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
ubuntu-2204-ci-deps:
name: Ubuntu 22.04
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: ubuntu-22.04
nox-session: ci-test-onedir
platform: linux
arch: x86_64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
ubuntu-2204-arm64-ci-deps:
name: Ubuntu 22.04 Arm64
needs:
- prepare-workflow
- download-onedir-artifact
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: ubuntu-22.04-arm64
nox-session: ci-test-onedir
platform: linux
arch: aarch64
nox-version: 2022.8.7
python-version: "3.10"
salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}"
cache-prefix: ${{ needs.prepare-workflow.outputs.cache-seed }}|3.10.13
backup:
name: Backup
runs-on:
@ -229,6 +741,34 @@ jobs:
needs:
- prepare-workflow
- publish-repositories
- almalinux-8-arm64-ci-deps
- almalinux-8-ci-deps
- almalinux-9-arm64-ci-deps
- almalinux-9-ci-deps
- amazonlinux-2-arm64-ci-deps
- amazonlinux-2-ci-deps
- centos-7-arm64-ci-deps
- centos-7-ci-deps
- centosstream-8-arm64-ci-deps
- centosstream-8-ci-deps
- centosstream-9-arm64-ci-deps
- centosstream-9-ci-deps
- debian-10-ci-deps
- debian-11-arm64-ci-deps
- debian-11-ci-deps
- fedora-37-arm64-ci-deps
- fedora-37-ci-deps
- fedora-38-arm64-ci-deps
- fedora-38-ci-deps
- macos-12-ci-deps
- photonos-3-ci-deps
- photonos-4-arm64-ci-deps
- photonos-4-ci-deps
- ubuntu-2004-arm64-ci-deps
- ubuntu-2004-ci-deps
- ubuntu-2204-arm64-ci-deps
- ubuntu-2204-ci-deps
- windows-2022-ci-deps
- download-onedir-artifact
uses: ./.github/workflows/test-package-downloads-action.yml
with:
@ -416,6 +956,38 @@ jobs:
- pkg-download-tests
- release
- publish-pypi
- windows-2016-ci-deps
- windows-2019-ci-deps
- windows-2022-ci-deps
- macos-12-ci-deps
- almalinux-8-ci-deps
- almalinux-8-arm64-ci-deps
- almalinux-9-ci-deps
- almalinux-9-arm64-ci-deps
- amazonlinux-2-ci-deps
- amazonlinux-2-arm64-ci-deps
- archlinux-lts-ci-deps
- centos-7-ci-deps
- centos-7-arm64-ci-deps
- centosstream-8-ci-deps
- centosstream-8-arm64-ci-deps
- centosstream-9-ci-deps
- centosstream-9-arm64-ci-deps
- debian-10-ci-deps
- debian-11-ci-deps
- debian-11-arm64-ci-deps
- fedora-37-ci-deps
- fedora-37-arm64-ci-deps
- fedora-38-ci-deps
- fedora-38-arm64-ci-deps
- opensuse-15-ci-deps
- photonos-3-ci-deps
- photonos-4-ci-deps
- photonos-4-arm64-ci-deps
- ubuntu-2004-ci-deps
- ubuntu-2004-arm64-ci-deps
- ubuntu-2204-ci-deps
- ubuntu-2204-arm64-ci-deps
steps:
- name: Get workflow information
id: get-workflow-info

28
.github/workflows/staging.yml vendored
View file

@ -2974,6 +2974,34 @@ jobs:
needs:
- prepare-workflow
- publish-repositories
- almalinux-8-arm64-ci-deps
- almalinux-8-ci-deps
- almalinux-9-arm64-ci-deps
- almalinux-9-ci-deps
- amazonlinux-2-arm64-ci-deps
- amazonlinux-2-ci-deps
- centos-7-arm64-ci-deps
- centos-7-ci-deps
- centosstream-8-arm64-ci-deps
- centosstream-8-ci-deps
- centosstream-9-arm64-ci-deps
- centosstream-9-ci-deps
- debian-10-ci-deps
- debian-11-arm64-ci-deps
- debian-11-ci-deps
- fedora-37-arm64-ci-deps
- fedora-37-ci-deps
- fedora-38-arm64-ci-deps
- fedora-38-ci-deps
- macos-12-ci-deps
- photonos-3-ci-deps
- photonos-4-arm64-ci-deps
- photonos-4-ci-deps
- ubuntu-2004-arm64-ci-deps
- ubuntu-2004-ci-deps
- ubuntu-2204-arm64-ci-deps
- ubuntu-2204-ci-deps
- windows-2022-ci-deps
- build-salt-onedir-linux
- build-salt-onedir-macos
- build-salt-onedir-windows

18
.github/workflows/templates/build-ci-deps.yml.jinja vendored
View file

@ -4,10 +4,16 @@
<{ slug.replace(".", "") }>-ci-deps:
<%- do test_salt_needs.append(slug.replace(".", "") + "-ci-deps") %>
name: <{ display_name }>
<%- if workflow_slug != 'release' %>
if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }}
<%- endif %>
needs:
- prepare-workflow
<%- if workflow_slug != 'release' %>
- build-salt-onedir-windows
<%- else %>
- download-onedir-artifact
<%- endif %>
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: <{ slug }>
@ -27,10 +33,16 @@
<{ slug.replace(".", "") }>-ci-deps:
<%- do test_salt_needs.append(slug.replace(".", "") + "-ci-deps") %>
name: <{ display_name }>
<%- if workflow_slug != 'release' %>
if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }}
<%- endif %>
needs:
- prepare-workflow
<%- if workflow_slug != 'release' %>
- build-salt-onedir-macos
<%- else %>
- download-onedir-artifact
<%- endif %>
uses: ./.github/workflows/build-deps-ci-action-macos.yml
with:
distro-slug: <{ slug }>
@ -50,10 +62,16 @@
<{ slug.replace(".", "") }>-ci-deps:
<%- do test_salt_needs.append(slug.replace(".", "") + "-ci-deps") %>
name: <{ display_name }>
<%- if workflow_slug != 'release' %>
if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }}
<%- endif %>
needs:
- prepare-workflow
<%- if workflow_slug != 'release' %>
- build-salt-onedir-linux
<%- else %>
- download-onedir-artifact
<%- endif %>
uses: ./.github/workflows/build-deps-ci-action.yml
with:
distro-slug: <{ slug }>

4
.github/workflows/templates/release.yml.jinja vendored
View file

@ -140,7 +140,7 @@ permissions:
- name: Set Cache Seed Output
id: set-cache-seed
run: |
echo "cache-seed=${{ env.CACHE_SEED }}" >> "$GITHUB_OUTPUT"
tools ci define-cache-seed ${{ env.CACHE_SEED }}
<%- endblock prepare_workflow_job %>
<%- endif %>
@ -196,6 +196,8 @@ permissions:
retention-days: 7
if-no-files-found: error
<%- include "build-ci-deps.yml.jinja" %>
backup:
name: Backup
runs-on:

737
.github/workflows/templates/test-package-downloads-action.yml.jinja vendored Normal file
View file

@ -0,0 +1,737 @@
name: Test Download Packages
on:
workflow_call:
inputs:
salt-version:
type: string
required: true
description: The Salt version of the packages to install and test
cache-prefix:
required: true
type: string
description: Seed used to invalidate caches
environment:
required: true
type: string
description: The environment to run tests against
latest-release:
required: true
type: string
description: The latest salt release
nox-version:
required: true
type: string
description: The nox version to install
python-version:
required: false
type: string
description: The python version to run tests with
default: "3.10"
package-name:
required: false
type: string
description: The onedir package name to use
default: salt
skip-code-coverage:
required: false
type: boolean
description: Skip code coverage
default: false
nox-session:
required: false
type: string
description: The nox session to run
default: ci-test-onedir
env:
COLUMNS: 190
AWS_MAX_ATTEMPTS: "10"
AWS_RETRY_MODE: "adaptive"
PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
PIP_EXTRA_INDEX_URL: https://pypi.org/simple
jobs:
linux:
name: Linux
runs-on:
- self-hosted
- linux
- bastion
environment: ${{ inputs.environment }}
timeout-minutes: 120 # 2 Hours - More than this and something is wrong
strategy:
fail-fast: false
matrix:
include:
<%- for slug, arch, pkg_type in test_salt_pkg_downloads_listing["linux"] %>
- distro-slug: <{ slug }>
arch: <{ arch }>
pkg-type: <{ pkg_type }>
<%- endfor %>
steps:
- name: "Throttle Builds"
shell: bash
run: |
t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"
- name: Checkout Source Code
uses: actions/checkout@v4
- name: Download Onedir Tarball as an Artifact
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-linux-${{ matrix.arch }}.tar.xz
path: artifacts/
- name: Decompress Onedir Tarball
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-linux-${{ matrix.arch }}.tar.xz
- name: Download cached nox.${{ matrix.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
uses: actions/cache@v3
with:
path: nox.${{ matrix.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|${{ matrix.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{
hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py')
}}
# If we get a cache miss here it means the dependencies step failed to save the cache
fail-on-cache-miss: true
- name: Setup Python Tools Scripts
uses: ./.github/actions/setup-python-tools-scripts
- name: Get Salt Project GitHub Actions Bot Environment
run: |
TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"
- name: Start VM
id: spin-up-vm
run: |
tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ matrix.distro-slug }}
- name: List Free Space
run: |
tools --timestamps vm ssh ${{ matrix.distro-slug }} -- df -h || true
- name: Upload Checkout To VM
run: |
tools --timestamps vm rsync ${{ matrix.distro-slug }}
- name: Decompress .nox Directory
run: |
tools --timestamps vm decompress-dependencies ${{ matrix.distro-slug }}
- name: Show System Info & Test Plan
env:
SALT_RELEASE: "${{ inputs.salt-version }}"
SALT_REPO_ARCH: ${{ matrix.arch }}
SALT_REPO_TYPE: ${{ inputs.environment }}
SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }}
SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }}
SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }}
SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }}
SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
LATEST_SALT_RELEASE: "${{ inputs.latest-release }}"
DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }}
run: |
tools --timestamps --timeout-secs=1800 vm testplan --skip-requirements-install \
-E INSTALL_TYPE -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \
-E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE -E DOWNLOAD_TEST_PACKAGE_TYPE \
--nox-session=${{ inputs.nox-session }}-pkgs ${{ matrix.distro-slug }} -- download-pkgs
- name: Run Package Download Tests
env:
SALT_RELEASE: "${{ inputs.salt-version }}"
SALT_REPO_ARCH: ${{ matrix.arch }}
SALT_REPO_TYPE: ${{ inputs.environment }}
SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }}
SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }}
SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }}
SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }}
SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
LATEST_SALT_RELEASE: "${{ inputs.latest-release }}"
DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }}
run: |
tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
-E INSTALL_TYPE -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \
-E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE -E DOWNLOAD_TEST_PACKAGE_TYPE \
--nox-session=${{ inputs.nox-session }}-pkgs --rerun-failures ${{ matrix.distro-slug }} -- download-pkgs
- name: Combine Coverage Reports
if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled'
run: |
tools --timestamps vm combine-coverage ${{ matrix.distro-slug }}
- name: Download Test Run Artifacts
id: download-artifacts-from-vm
if: always() && steps.spin-up-vm.outcome == 'success'
run: |
tools --timestamps vm download-artifacts ${{ matrix.distro-slug }}
# Delete the salt onedir, we won't need it anymore and it will prevent
# from it showing in the tree command below
rm -rf artifacts/salt*
tree -a artifacts
- name: Destroy VM
if: always()
run: |
tools --timestamps vm destroy --no-wait ${{ matrix.distro-slug }} || true
- name: Fix file ownership
run: |
sudo chown -R "$(id -un)" .
- name: Install Codecov CLI
if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
run: |
# We can't yet use tokenless uploads with the codecov CLI
# python3 -m pip install codecov-cli
#
curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --import
curl -Os https://uploader.codecov.io/latest/linux/codecov
curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM
curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig
gpg --verify codecov.SHA256SUM.sig codecov.SHA256SUM
shasum -a 256 -c codecov.SHA256SUM
chmod +x codecov
- name: Upload Source Code Coverage To Codecov
if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
run: |
if [ ! -s artifacts/coverage/salt.xml ]; then
echo "The artifacts/coverage/salt.xml file does not exist"
exit 1
fi
# We can't yet use tokenless uploads with the codecov CLI
#codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \
# do-upload --git-service github --sha ${{ github.sha }} \
# --file artifacts/coverage/salt.xml \
# --flag salt --flag ${{ matrix.distro-slug }} --flag pkg \
# --name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs
n=0
until [ "$n" -ge 5 ]
do
if ./codecov --file artifacts/coverage/salt.xml \
--sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \
--flags salt,${{ matrix.distro-slug }},pkg \
--name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then
rc=$?
break
fi
rc=$?
n=$((n+1))
sleep 15
done
if [ "$rc" -ne 0 ]; then
echo "Failed to upload codecov stats"
exit 1
fi
- name: Upload Tests Code Coverage To Codecov
if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
run: |
if [ ! -s artifacts/coverage/tests.xml ]; then
echo "The artifacts/coverage/tests.xml file does not exist"
exit 1
fi
# We can't yet use tokenless uploads with the codecov CLI
#codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \
# do-upload --git-service github --sha ${{ github.sha }} \
# --file artifacts/coverage/tests.xml \
# --flag tests --flag ${{ matrix.distro-slug }} --flag pkg \
# --name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs
n=0
until [ "$n" -ge 5 ]
do
if ./codecov --file artifacts/coverage/tests.xml \
--sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \
--flags tests,${{ matrix.distro-slug }},pkg \
--name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then
rc=$?
break
fi
rc=$?
n=$((n+1))
sleep 15
done
if [ "$rc" -ne 0 ]; then
echo "Failed to upload codecov stats"
exit 1
fi
- name: Upload Test Run Artifacts
if: always() && steps.download-artifacts-from-vm.outcome == 'success'
uses: actions/upload-artifact@v3
with:
name: pkg-testrun-artifacts-${{ matrix.distro-slug }}-${{ matrix.arch }}
path: |
artifacts
!artifacts/salt/*
!artifacts/salt-*.tar.*
- name: Publish Test Report
uses: mikepenz/action-junit-report@v3
# always run even if the previous steps fails
if: always() && job.status != 'cancelled' && steps.download-artifacts-from-vm.outcome == 'success'
with:
check_name: Overall Test Results(${{ matrix.distro-slug }} ${{ matrix.arch }})
report_paths: 'artifacts/xml-unittests-output/*.xml'
annotate_only: true
macos:
name: MacOS
runs-on: ${{ matrix.distro-slug }}
environment: ${{ inputs.environment }}
timeout-minutes: 120 # 2 Hours - More than this and something is wrong
strategy:
fail-fast: false
matrix:
include:
<%- for slug, arch, pkg_type in test_salt_pkg_downloads_listing["macos"] %>
- distro-slug: <{ slug }>
arch: <{ arch }>
pkg-type: <{ pkg_type }>
<%- endfor %>
steps:
- name: "Throttle Builds"
shell: bash
run: |
t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t"
- name: Checkout Source Code
uses: actions/checkout@v4
- name: Download Onedir Tarball as an Artifact
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-macos-${{ matrix.arch }}.tar.xz
path: artifacts/
- name: Install System Dependencies
run: |
brew install tree
- name: Decompress Onedir Tarball
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-macos-${{ matrix.arch }}.tar.xz
- name: Set up Python ${{ inputs.python-version }}
uses: actions/setup-python@v4
with:
python-version: "${{ inputs.python-version }}"
update-environment: true
- name: Install Nox
run: |
python3 -m pip install 'nox==${{ inputs.nox-version }}'
- name: Download cached nox.${{ matrix.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
uses: actions/cache@v3
with:
path: nox.${{ matrix.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|${{ matrix.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{
hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py')
}}
# If we get a cache miss here it means the dependencies step failed to save the cache
fail-on-cache-miss: true
- name: Decompress .nox Directory
run: |
nox --force-color -e decompress-dependencies -- ${{ matrix.distro-slug }}
- name: Show System Info & Test Plan
env:
SALT_RELEASE: "${{ inputs.salt-version }}"
SKIP_REQUIREMENTS_INSTALL: "1"
PRINT_TEST_SELECTION: "1"
PRINT_TEST_PLAN_ONLY: "1"
PRINT_SYSTEM_INFO: "1"
GITHUB_ACTIONS_PIPELINE: "1"
SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
LATEST_SALT_RELEASE: "${{ inputs.latest-release }}"
DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }}
run: |
sudo -E nox --force-color -e ${{ inputs.nox-session }}-pkgs -- download-pkgs
- name: Run Package Download Tests
env:
SKIP_REQUIREMENTS_INSTALL: "1"
PRINT_TEST_SELECTION: "0"
PRINT_TEST_PLAN_ONLY: "0"
PRINT_SYSTEM_INFO: "0"
RERUN_FAILURES: "1"
GITHUB_ACTIONS_PIPELINE: "1"
SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
COVERAGE_CONTEXT: ${{ matrix.distro-slug }}
SALT_RELEASE: "${{ inputs.salt-version }}"
SALT_REPO_ARCH: ${{ matrix.arch }}
LATEST_SALT_RELEASE: "${{ inputs.latest-release }}"
SALT_REPO_TYPE: ${{ inputs.environment }}
SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }}
SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }}
SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }}
SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }}
DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }}
run: |
sudo -E nox --force-color -e ${{ inputs.nox-session }}-pkgs -- download-pkgs
- name: Fix file ownership
run: |
sudo chown -R "$(id -un)" .
- name: Combine Coverage Reports
if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled'
run: |
nox --force-color -e combine-coverage
- name: Prepare Test Run Artifacts
id: download-artifacts-from-vm
if: always() && job.status != 'cancelled'
run: |
# Delete the salt onedir, we won't need it anymore and it will prevent
# from it showing in the tree command below
rm -rf artifacts/salt*
tree -a artifacts
- name: Install Codecov CLI
if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled'
run: |
# We can't yet use tokenless uploads with the codecov CLI
# python3 -m pip install codecov-cli
#
curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --import
curl -Os https://uploader.codecov.io/latest/macos/codecov
curl -Os https://uploader.codecov.io/latest/macos/codecov.SHA256SUM
curl -Os https://uploader.codecov.io/latest/macos/codecov.SHA256SUM.sig
gpg --verify codecov.SHA256SUM.sig codecov.SHA256SUM
shasum -a 256 -c codecov.SHA256SUM
chmod +x codecov
- name: Upload Source Code Coverage To Codecov
if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled'
run: |
if [ ! -s artifacts/coverage/salt.xml ]; then
echo "The artifacts/coverage/salt.xml file does not exist"
exit 1
fi
# We can't yet use tokenless uploads with the codecov CLI
#codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \
# do-upload --git-service github --sha ${{ github.sha }} \
# --file artifacts/coverage/salt.xml \
# --flag salt --flag ${{ matrix.distro-slug }} --flag pkg \
# --name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs
n=0
until [ "$n" -ge 5 ]
do
if ./codecov --file artifacts/coverage/salt.xml \
--sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \
--flags salt,${{ matrix.distro-slug }},pkg \
--name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then
rc=$?
break
fi
rc=$?
n=$((n+1))
sleep 15
done
if [ "$rc" -ne 0 ]; then
echo "Failed to upload codecov stats"
exit 1
fi
- name: Upload Tests Code Coverage To Codecov
if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled'
run: |
if [ ! -s artifacts/coverage/tests.xml ]; then
echo "The artifacts/coverage/tests.xml file does not exist"
exit 1
fi
# We can't yet use tokenless uploads with the codecov CLI
#codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \
# do-upload --git-service github --sha ${{ github.sha }} \
# --file artifacts/coverage/tests.xml \
# --flag tests --flag ${{ matrix.distro-slug }} --flag pkg \
# --name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs
n=0
until [ "$n" -ge 5 ]
do
if ./codecov --file artifacts/coverage/tests.xml \
--sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \
--flags tests,${{ matrix.distro-slug }},pkg \
--name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then
rc=$?
break
fi
rc=$?
n=$((n+1))
sleep 15
done
if [ "$rc" -ne 0 ]; then
echo "Failed to upload codecov stats"
exit 1
fi
- name: Upload Test Run Artifacts
if: always()
uses: actions/upload-artifact@v3
with:
name: pkg-testrun-artifacts-${{ matrix.distro-slug }}-${{ matrix.arch }}
path: |
artifacts
!artifacts/salt/*
!artifacts/salt-*.tar.*
- name: Publish Test Report
uses: mikepenz/action-junit-report@v3
# always run even if the previous steps fails
if: always() && job.status != 'cancelled'
with:
check_name: Overall Test Results(${{ matrix.distro-slug }} ${{ matrix.arch }})
report_paths: 'artifacts/xml-unittests-output/*.xml'
annotate_only: true
windows:
name: Windows
runs-on:
- self-hosted
- linux
- bastion
environment: ${{ inputs.environment }}
timeout-minutes: 120 # 2 Hours - More than this and something is wrong
strategy:
fail-fast: false
matrix:
include:
<%- for slug, arch, pkg_type in test_salt_pkg_downloads_listing["windows"] %>
- distro-slug: <{ slug }>
arch: <{ arch }>
pkg-type: <{ pkg_type }>
<%- endfor %>
steps:
- name: Checkout Source Code
uses: actions/checkout@v4
- name: Download Onedir Tarball as an Artifact
uses: actions/download-artifact@v3
with:
name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-windows-${{ matrix.arch }}.tar.xz
path: artifacts/
- name: Decompress Onedir Tarball
shell: bash
run: |
python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
cd artifacts
tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-windows-${{ matrix.arch }}.tar.xz
- name: Download cached nox.${{ matrix.distro-slug }}.tar.* for session ${{ inputs.nox-session }}
uses: actions/cache@v3
with:
path: nox.${{ matrix.distro-slug }}.tar.*
key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|${{ matrix.distro-slug }}|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{
hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py')
}}
# If we get a cache miss here it means the dependencies step failed to save the cache
fail-on-cache-miss: true
- name: Setup Python Tools Scripts
uses: ./.github/actions/setup-python-tools-scripts
- name: Get Salt Project GitHub Actions Bot Environment
run: |
TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"
- name: Start VM
id: spin-up-vm
run: |
tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ matrix.distro-slug }}
- name: List Free Space
run: |
tools --timestamps vm ssh ${{ matrix.distro-slug }} -- df -h || true
- name: Upload Checkout To VM
run: |
tools --timestamps vm rsync ${{ matrix.distro-slug }}
- name: Decompress .nox Directory
run: |
tools --timestamps vm decompress-dependencies ${{ matrix.distro-slug }}
- name: Show System Info & Test Plan
env:
SALT_RELEASE: "${{ inputs.salt-version }}"
SALT_REPO_ARCH: ${{ matrix.arch }}
LATEST_SALT_RELEASE: "${{ inputs.latest-release }}"
SALT_REPO_TYPE: ${{ inputs.environment }}
SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }}
SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }}
SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }}
SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }}
SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }}
run: |
tools --timestamps --timeout-secs=1800 vm testplan --skip-requirements-install \
-E INSTALL_TYPE -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \
-E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE -E DOWNLOAD_TEST_PACKAGE_TYPE \
--nox-session=${{ inputs.nox-session }}-pkgs ${{ matrix.distro-slug }} -- download-pkgs
- name: Run Package Download Tests
env:
SALT_RELEASE: "${{ inputs.salt-version }}"
SALT_REPO_ARCH: ${{ matrix.arch }}
LATEST_SALT_RELEASE: "${{ inputs.latest-release }}"
SALT_REPO_TYPE: ${{ inputs.environment }}
SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }}
SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }}
SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }}
SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }}
SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }}
run: |
tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
-E INSTALL_TYPE -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \
-E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE -E DOWNLOAD_TEST_PACKAGE_TYPE \
--nox-session=${{ inputs.nox-session }}-pkgs --rerun-failures ${{ matrix.distro-slug }} -- download-pkgs
- name: Combine Coverage Reports
if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled'
run: |
tools --timestamps vm combine-coverage ${{ matrix.distro-slug }}
- name: Download Test Run Artifacts
id: download-artifacts-from-vm
if: always() && steps.spin-up-vm.outcome == 'success'
run: |
tools --timestamps vm download-artifacts ${{ matrix.distro-slug }}
# Delete the salt onedir, we won't need it anymore and it will prevent
# from it showing in the tree command below
rm -rf artifacts/salt*
tree -a artifacts
- name: Destroy VM
if: always()
run: |
tools --timestamps vm destroy --no-wait ${{ matrix.distro-slug }} || true
- name: Fix file ownership
run: |
sudo chown -R "$(id -un)" .
- name: Install Codecov CLI
if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
run: |
# We can't yet use tokenless uploads with the codecov CLI
# python3 -m pip install codecov-cli
#
curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --import
curl -Os https://uploader.codecov.io/latest/linux/codecov
curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM
curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig
gpg --verify codecov.SHA256SUM.sig codecov.SHA256SUM
shasum -a 256 -c codecov.SHA256SUM
chmod +x codecov
- name: Upload Source Code Coverage To Codecov
if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
run: |
if [ ! -s artifacts/coverage/salt.xml ]; then
echo "The artifacts/coverage/salt.xml file does not exist"
exit 1
fi
# We can't yet use tokenless uploads with the codecov CLI
#codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \
# do-upload --git-service github --sha ${{ github.sha }} \
# --file artifacts/coverage/salt.xml \
# --flag salt --flag ${{ matrix.distro-slug }} --flag pkg \
# --name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs
n=0
until [ "$n" -ge 5 ]
do
if ./codecov --file artifacts/coverage/salt.xml \
--sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \
--flags salt,${{ matrix.distro-slug }},pkg \
--name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then
rc=$?
break
fi
rc=$?
n=$((n+1))
sleep 15
done
if [ "$rc" -ne 0 ]; then
echo "Failed to upload codecov stats"
exit 1
fi
- name: Upload Tests Code Coverage To Codecov
if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
run: |
if [ ! -s artifacts/coverage/tests.xml ]; then
echo "The artifacts/coverage/tests.xml file does not exist"
exit 1
fi
# We can't yet use tokenless uploads with the codecov CLI
#codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \
# do-upload --git-service github --sha ${{ github.sha }} \
# --file artifacts/coverage/tests.xml \
# --flag tests --flag ${{ matrix.distro-slug }} --flag pkg \
# --name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs
n=0
until [ "$n" -ge 5 ]
do
if ./codecov --file artifacts/coverage/tests.xml \
--sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \
--flags tests,${{ matrix.distro-slug }},pkg \
--name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then
rc=$?
break
fi
rc=$?
n=$((n+1))
sleep 15
done
if [ "$rc" -ne 0 ]; then
echo "Failed to upload codecov stats"
exit 1
fi
- name: Upload Test Run Artifacts
if: always() && steps.download-artifacts-from-vm.outcome == 'success'
uses: actions/upload-artifact@v3
with:
name: pkg-testrun-artifacts-${{ matrix.distro-slug }}-${{ matrix.arch }}
path: |
artifacts
!artifacts/salt/*
!artifacts/salt-*.tar.*
- name: Publish Test Report
uses: mikepenz/action-junit-report@v3
# always run even if the previous steps fails
if: always() && job.status != 'cancelled' && steps.download-artifacts-from-vm.outcome == 'success'
with:
check_name: Overall Test Results(${{ matrix.distro-slug }} ${{ matrix.arch }} ${{ matrix.pkg-type }} )
report_paths: 'artifacts/xml-unittests-output/*.xml'
annotate_only: true

3
.github/workflows/templates/test-salt-pkg-repo-downloads.yml.jinja vendored
View file

@ -13,6 +13,9 @@
needs:
- prepare-workflow
- publish-repositories
<%- for slug in test_salt_pkg_downloads_needs_slugs %>
- <{ slug }>
<%- endfor %>
<%- if gh_environment == "release" %>
- download-onedir-artifact
<%- else %>

209
.github/workflows/test-package-downloads-action.yml vendored
View file

@ -53,32 +53,6 @@ env:
jobs:
generate-matrix:
name: Test Matrix
runs-on:
- self-hosted
- linux
- x86_64
outputs:
matrix-include: ${{ steps.generate-matrix.outputs.matrix }}
steps:
- name: "Throttle Builds"
shell: bash
run: |
t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"
- name: Checkout Source Code
uses: actions/checkout@v4
- name: Setup Python Tools Scripts
uses: ./.github/actions/setup-python-tools-scripts
- name: Generate Test Matrix
id: generate-matrix
run: |
tools ci pkg-download-matrix
linux:
name: Linux
runs-on:
@ -87,12 +61,166 @@ jobs:
- bastion
environment: ${{ inputs.environment }}
timeout-minutes: 120 # 2 Hours - More than this and something is wrong
needs:
- generate-matrix
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['linux'] }}
include:
- distro-slug: almalinux-8
arch: x86_64
pkg-type: package
- distro-slug: almalinux-8
arch: x86_64
pkg-type: onedir
- distro-slug: almalinux-8-arm64
arch: aarch64
pkg-type: package
- distro-slug: almalinux-8-arm64
arch: aarch64
pkg-type: onedir
- distro-slug: almalinux-9
arch: x86_64
pkg-type: package
- distro-slug: almalinux-9
arch: x86_64
pkg-type: onedir
- distro-slug: almalinux-9-arm64
arch: aarch64
pkg-type: package
- distro-slug: almalinux-9-arm64
arch: aarch64
pkg-type: onedir
- distro-slug: amazonlinux-2
arch: x86_64
pkg-type: package
- distro-slug: amazonlinux-2
arch: x86_64
pkg-type: onedir
- distro-slug: amazonlinux-2-arm64
arch: aarch64
pkg-type: package
- distro-slug: amazonlinux-2-arm64
arch: aarch64
pkg-type: onedir
- distro-slug: centos-7
arch: x86_64
pkg-type: package
- distro-slug: centos-7
arch: x86_64
pkg-type: onedir
- distro-slug: centos-7-arm64
arch: aarch64
pkg-type: package
- distro-slug: centos-7-arm64
arch: aarch64
pkg-type: onedir
- distro-slug: centosstream-8
arch: x86_64
pkg-type: package
- distro-slug: centosstream-8
arch: x86_64
pkg-type: onedir
- distro-slug: centosstream-8-arm64
arch: aarch64
pkg-type: package
- distro-slug: centosstream-8-arm64
arch: aarch64
pkg-type: onedir
- distro-slug: centosstream-9
arch: x86_64
pkg-type: package
- distro-slug: centosstream-9
arch: x86_64
pkg-type: onedir
- distro-slug: centosstream-9-arm64
arch: aarch64
pkg-type: package
- distro-slug: centosstream-9-arm64
arch: aarch64
pkg-type: onedir
- distro-slug: debian-10
arch: x86_64
pkg-type: package
- distro-slug: debian-10
arch: x86_64
pkg-type: onedir
- distro-slug: debian-11
arch: x86_64
pkg-type: package
- distro-slug: debian-11
arch: x86_64
pkg-type: onedir
- distro-slug: debian-11-arm64
arch: aarch64
pkg-type: package
- distro-slug: debian-11-arm64
arch: aarch64
pkg-type: onedir
- distro-slug: fedora-37
arch: x86_64
pkg-type: package
- distro-slug: fedora-37
arch: x86_64
pkg-type: onedir
- distro-slug: fedora-37-arm64
arch: aarch64
pkg-type: package
- distro-slug: fedora-37-arm64
arch: aarch64
pkg-type: onedir
- distro-slug: fedora-38
arch: x86_64
pkg-type: package
- distro-slug: fedora-38
arch: x86_64
pkg-type: onedir
- distro-slug: fedora-38-arm64
arch: aarch64
pkg-type: package
- distro-slug: fedora-38-arm64
arch: aarch64
pkg-type: onedir
- distro-slug: photonos-3
arch: x86_64
pkg-type: package
- distro-slug: photonos-3
arch: x86_64
pkg-type: onedir
- distro-slug: photonos-4
arch: x86_64
pkg-type: package
- distro-slug: photonos-4
arch: x86_64
pkg-type: onedir
- distro-slug: photonos-4-arm64
arch: aarch64
pkg-type: package
- distro-slug: photonos-4-arm64
arch: aarch64
pkg-type: onedir
- distro-slug: ubuntu-20.04
arch: x86_64
pkg-type: package
- distro-slug: ubuntu-20.04
arch: x86_64
pkg-type: onedir
- distro-slug: ubuntu-20.04-arm64
arch: aarch64
pkg-type: package
- distro-slug: ubuntu-20.04-arm64
arch: aarch64
pkg-type: onedir
- distro-slug: ubuntu-22.04
arch: x86_64
pkg-type: package
- distro-slug: ubuntu-22.04
arch: x86_64
pkg-type: onedir
- distro-slug: ubuntu-22.04-arm64
arch: aarch64
pkg-type: package
- distro-slug: ubuntu-22.04-arm64
arch: aarch64
pkg-type: onedir
steps:
@ -316,12 +444,16 @@ jobs:
runs-on: ${{ matrix.distro-slug }}
environment: ${{ inputs.environment }}
timeout-minutes: 120 # 2 Hours - More than this and something is wrong
needs:
- generate-matrix
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['macos'] }}
include:
- distro-slug: macos-12
arch: x86_64
pkg-type: package
- distro-slug: macos-12
arch: x86_64
pkg-type: onedir
steps:
@ -536,12 +668,19 @@ jobs:
- bastion
environment: ${{ inputs.environment }}
timeout-minutes: 120 # 2 Hours - More than this and something is wrong
needs:
- generate-matrix
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['windows'] }}
include:
- distro-slug: windows-2022
arch: amd64
pkg-type: nsis
- distro-slug: windows-2022
arch: amd64
pkg-type: msi
- distro-slug: windows-2022
arch: amd64
pkg-type: onedir
steps:
- name: Checkout Source Code

68
tools/ci.py
View file

@ -777,74 +777,6 @@ def pkg_matrix(
ctx.exit(0)
@ci.command(
name="pkg-download-matrix",
arguments={},
)
def pkg_download_matrix(ctx: Context):
"""
Generate the test matrix.
"""
_platforms = {}
linux_entries = (
("almalinux-8", "Alma Linux 8", "x86_64", "package"),
("almalinux-8-arm64", "Alma Linux 8 Arm64", "aarch64", "package"),
("almalinux-9", "Alma Linux 9", "x86_64", "package"),
("almalinux-9-arm64", "Alma Linux 9 Arm64", "aarch64", "package"),
("amazonlinux-2", "Amazon Linux 2", "x86_64", "package"),
("amazonlinux-2-arm64", "Amazon Linux 2 Arm64", "aarch64", "package"),
("centos-7", "CentOS 7", "x86_64", "package"),
("centos-7-arm64", "CentOS 7 Arm64", "aarch64", "package"),
("centosstream-8", "CentOS Stream 8", "x86_64", "package"),
("centosstream-8-arm64", "CentOS Stream 8 Arm64", "aarch64", "package"),
("centosstream-9", "CentOS Stream 9", "x86_64", "package"),
("centosstream-9-arm64", "CentOS Stream 9 Arm64", "aarch64", "package"),
("debian-10", "Debian 10", "x86_64", "package"),
("debian-11", "Debian 11", "x86_64", "package"),
("debian-11-arm64", "Debian 11 Arm64", "aarch64", "package"),
("fedora-37", "Fedora 37", "x86_64", "package"),
("fedora-37-arm64", "Fedora 37 Arm64", "aarch64", "package"),
("fedora-38", "Fedora 38", "x86_64", "package"),
("fedora-38-arm64", "Fedora 38 Arm64", "aarch64", "package"),
("photonos-3", "Photon OS 3", "x86_64", "package"),
("photonos-4", "Photon OS 4", "x86_64", "package"),
("photonos-4-arm64", "Photon OS 4 Arm64", "aarch64", "package"),
("ubuntu-20.04", "Ubuntu 20.04", "x86_64", "package"),
("ubuntu-20.04-arm64", "Ubuntu 20.04 Arm64", "aarch64", "package"),
("ubuntu-22.04", "Ubuntu 22.04", "x86_64", "package"),
("ubuntu-22.04-arm64", "Ubuntu 22.04 Arm64", "aarch64", "package"),
("ubuntu-22.04", "Ubuntu 22.04", "x86_64", "onedir"),
("ubuntu-22.04-arm64", "Ubuntu 22.04 Arm64", "aarch64", "onedir"),
)
macos_entries = (
("macos-12", "macOS 12", "x86_64", "Package"),
("macos-12", "macOS 12", "x86_64", "Onedir"),
)
windows_entries = (("windows-2022", "Windows 2022", "amd64"),)
_matrix = []
for distro, display_name, arch, pkg_type in linux_entries:
_matrix.append({"distro-slug": distro, "arch": arch, "pkg_type": pkg_type})
_platforms["linux"] = _matrix
_matrix = []
for distro, display_name, arch, pkg_type in macos_entries:
_matrix.append({"distro-slug": distro, "arch": arch, "pkg_type": pkg_type})
_platforms["macos"] = _matrix
_matrix = []
for distro, display_name, arch in windows_entries:
for pkg_type in ("nsis", "msi", "onedir"):
_matrix.append({"distro-slug": distro, "arch": arch, "pkg_type": pkg_type})
_platforms["windows"] = _matrix
ctx.info("Generated matrix:")
ctx.print(_platforms, soft_wrap=True)
github_output = os.environ.get("GITHUB_OUTPUT")
if github_output is not None:
with open(github_output, "a", encoding="utf-8") as wfh:
wfh.write(f"matrix={json.dumps(_platforms)}\n")
ctx.exit(0)
@ci.command(
name="get-releases",
arguments={

41
tools/pre_commit.py
View file

@ -80,6 +80,9 @@ def generate_workflows(ctx: Context):
"test-pkg-downloads": True,
},
},
"Test Package Downloads": {
"template": "test-package-downloads-action.yml",
},
}
test_salt_listing = {
"linux": (
@ -136,7 +139,7 @@ def generate_workflows(ctx: Context):
),
}
build_ci_deps_listing = {
"linux": (
"linux": [
("almalinux-8", "Alma Linux 8", "x86_64"),
("almalinux-8-arm64", "Alma Linux 8 Arm64", "aarch64"),
("almalinux-9", "Alma Linux 9", "x86_64"),
@ -165,14 +168,38 @@ def generate_workflows(ctx: Context):
("ubuntu-20.04-arm64", "Ubuntu 20.04 Arm64", "aarch64"),
("ubuntu-22.04", "Ubuntu 22.04", "x86_64"),
("ubuntu-22.04-arm64", "Ubuntu 22.04 Arm64", "aarch64"),
),
"macos": (("macos-12", "macOS 12", "x86_64"),),
"windows": (
],
"macos": [
("macos-12", "macOS 12", "x86_64"),
],
"windows": [
("windows-2016", "Windows 2016", "amd64"),
("windows-2019", "Windows 2019", "amd64"),
("windows-2022", "Windows 2022", "amd64"),
),
],
}
test_salt_pkg_downloads_listing: dict[str, list[tuple[str, str, str]]] = {
"linux": [],
"macos": [],
"windows": [],
}
for slug, display_name, arch in build_ci_deps_listing["linux"]:
if slug in ("archlinux-lts", "opensuse-15"):
continue
for pkg_type in ("package", "onedir"):
test_salt_pkg_downloads_listing["linux"].append((slug, arch, pkg_type))
for slug, display_name, arch in build_ci_deps_listing["macos"]:
for pkg_type in ("package", "onedir"):
test_salt_pkg_downloads_listing["macos"].append((slug, arch, pkg_type))
for slug, display_name, arch in build_ci_deps_listing["windows"][-1:]:
for pkg_type in ("nsis", "msi", "onedir"):
test_salt_pkg_downloads_listing["windows"].append((slug, arch, pkg_type))
test_salt_pkg_downloads_needs_slugs = set()
for platform in test_salt_pkg_downloads_listing:
for slug, _, _ in test_salt_pkg_downloads_listing[platform]:
test_salt_pkg_downloads_needs_slugs.add(f"{slug.replace('.', '')}-ci-deps")
env = Environment(
block_start_string="<%",
block_end_string="%>",
@ -210,6 +237,10 @@ def generate_workflows(ctx: Context):
"test_salt_listing": test_salt_listing,
"test_salt_pkg_listing": test_salt_pkg_listing,
"build_ci_deps_listing": build_ci_deps_listing,
"test_salt_pkg_downloads_listing": test_salt_pkg_downloads_listing,
"test_salt_pkg_downloads_needs_slugs": sorted(
test_salt_pkg_downloads_needs_slugs
),
}
shared_context_file = (
tools.utils.REPO_ROOT / "cicd" / "shared-gh-workflows-context.yml"