diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28d0bb5ceda..1ca7ad204dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,28 +1,10 @@ +# Do not edit these workflows directly as the changes made will be overwritten. +# Instead, edit the template '.github/workflows/templates/ci.yml.j2' +--- name: CI - on: push: {} pull_request: {} - schedule: - # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onschedule - - cron: '0 */8 * * *' # Run every 8 hours - workflow_dispatch: - inputs: - salt-version: - type: string - default: "" - description: > - The Salt version to set prior to running tests or building packages. - If not set, it is discover at run time, like, for example, capturing - the output of running `python3 salt/version.py` - testrun-type: - description: Type of test run - required: true - default: 'full' - type: choice - options: - - full - - changed env: COLUMNS: 160 @@ -38,7 +20,7 @@ concurrency: # not cancel previous builds. # However, for every new build against the same pull request source branch, # all older builds against that same branch get canceled. - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: @@ -134,7 +116,7 @@ jobs: id: setup-salt-version uses: ./.github/actions/setup-salt-version with: - salt-version: "${{ inputs.salt-version }}" + salt-version: "" - name: Write Changed Files To A Local File if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}} @@ -220,7 +202,7 @@ jobs: changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} prepare-release: - name: Create Release Diff + name: Prepare Release if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} needs: - prepare-workflow @@ -296,7 +278,6 @@ jobs: self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} - windows-2016: name: Windows 2016 if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} @@ -553,7 +534,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} - photon-3: + photonos-3: name: Photon OS 3 if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} needs: @@ -569,7 +550,7 @@ jobs: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} - photon-4: + photonos-4: name: Photon OS 4 if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} needs: @@ -672,12 +653,17 @@ jobs: if: always() runs-on: ubuntu-latest needs: + - prepare-workflow - pre-commit - docs - lint - build-deps-onedir - build-salt-onedir - build-pkgs + - windows-2016 + - windows-2019 + - windows-2022 + - macos-12 - almalinux-8 - almalinux-9 - amazonlinux-2 @@ -690,17 +676,13 @@ jobs: - debian-11-arm64 - fedora-36 - opensuse-15 - - photon-3 - - photon-4 + - photonos-3 + - photonos-4 - ubuntu-1804 - ubuntu-2004 - ubuntu-2004-arm64 - ubuntu-2204 - ubuntu-2204-arm64 - - windows-2016 - - windows-2019 - - windows-2022 - - macos-12 steps: - name: Get workflow information id: get-workflow-info diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index cf731fa0479..1f284de39a2 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -23,7 +23,7 @@ env: jobs: build: - name: Prepare Release + name: Create Release Diff runs-on: ubuntu-latest steps: diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml new file mode 100644 index 00000000000..6d104a1dac2 --- /dev/null +++ b/.github/workflows/scheduled.yml @@ -0,0 +1,701 @@ +# Do not edit these workflows directly as the changes made will be overwritten. +# Instead, edit the template '.github/workflows/templates/scheduled.yml.j2' +--- +name: Scheduled + +on: + schedule: + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onschedule + - cron: '0 */8 * * *' # Run every 8 hours + +env: + COLUMNS: 160 + CACHE_SEED: SEED-4 # Bump the number to invalidate all caches + RELENV_DATA: "${{ github.workspace }}/.relenv" + +permissions: + contents: read # for dorny/paths-filter to fetch a list of changed files + pull-requests: read # for dorny/paths-filter to read pull requests + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: false + +jobs: + prepare-workflow: + name: Prepare Workflow Run + runs-on: ubuntu-latest + outputs: + jobs: ${{ steps.define-jobs.outputs.jobs }} + changed-files: ${{ steps.process-changed-files.outputs.changed-files }} + testrun: ${{ steps.define-testrun.outputs.testrun }} + salt-version: ${{ steps.setup-salt-version.outputs.salt-version }} + cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Full clone to also get the tags to get the right salt version + + - name: Get Changed Files + if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}} + id: changed-files + uses: dorny/paths-filter@v2 + with: + token: ${{ github.token }} + list-files: json + filters: | + repo: + - added|modified: + - '**' + doc-requirements: + - added|modified: &doc_requirements + - requirements/static/ci/py3.*/docs.txt + lint-requirements: + - added|modified: &lint_requirements + - requirements/static/ci/py3.*/lint.txt + pkg_requirements: + - added|modified: &pkg_requirements + - requirements/static/pkg/py3.*/darwin.txt + - requirements/static/pkg/py3.*/linux.txt + - requirements/static/pkg/py3.*/freebsd.txt + - requirements/static/pkg/py3.*/windows.txt + test_requirements: + - added|modified: &test_requirements + - requirements/static/ci/py3.*/darwin.txt + - requirements/static/ci/py3.*/linux.txt + - requirements/static/ci/py3.*/freebsd.txt + - requirements/static/ci/py3.*/windows.txt + - requirements/static/ci/py3.*/darwin-crypto.txt + - requirements/static/ci/py3.*/linux-crypto.txt + - requirements/static/ci/py3.*/freebsd-crypto.txt + - requirements/static/ci/py3.*/windows-crypto.txt + deleted: + - deleted: + - '**' + docs: + - added|modified: + - doc/** + - *doc_requirements + salt: + - added|modified: &salt_added_modified + - setup.py + - noxfile.py + - salt/**/*.py + - tasks/**/*.py + - tools/**/*.py + tests: + - added|modified: &tests_added_modified + - tests/**/*.py + lint: + - added|modified: + - .pylintrc + - *lint_requirements + golden_images: + - added|modified: + - cicd/golden-images.json + testrun: + - added|modified: + - *salt_added_modified + - *tests_added_modified + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Setup Python Tools Scripts + uses: ./.github/actions/setup-python-tools-scripts + + - name: Pretty Print The GH Actions Event + run: + tools ci print-gh-event + + - name: Setup Salt Version + id: setup-salt-version + uses: ./.github/actions/setup-salt-version + with: + salt-version: "" + + - name: Write Changed Files To A Local File + if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}} + run: + echo '${{ toJSON(steps.changed-files.outputs) }}' > changed-files.json + + - name: Check Local Changed Files Contents + if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}} + run: + cat changed-files.json + + - name: Process Changed Files + if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}} + id: process-changed-files + run: + tools ci process-changed-files ${{ github.event_name }} changed-files.json + + - name: Check Collected Changed Files + if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}} + run: + echo '${{ steps.process-changed-files.outputs.changed-files }}' | jq -C '.' + + - name: Define Jobs To Run + id: define-jobs + run: + tools ci define-jobs ${{ github.event_name }} + + - name: Check Collected Jobs + run: + echo '${{ steps.define-jobs.outputs.jobs }}' | jq -C '.' + + - name: Define Testrun + id: define-testrun + run: + tools ci define-testrun ${{ github.event_name }} changed-files.json + + - name: Check Defined Test Run + run: + echo '${{ steps.define-testrun.outputs.testrun }}' | jq -C '.' + + - name: Check Contents of generated testrun-changed-files.txt + if: ${{ fromJSON(steps.define-testrun.outputs.testrun)['type'] != 'full' }} + run: + cat testrun-changed-files.txt || true + + - name: Upload testrun-changed-files.txt + if: ${{ fromJSON(steps.define-testrun.outputs.testrun)['type'] != 'full' }} + uses: actions/upload-artifact@v3 + with: + name: testrun-changed-files.txt + path: testrun-changed-files.txt + + - name: Set Cache Seed Output + id: set-cache-seed + run: | + echo "cache-seed=${{ env.CACHE_SEED }}" >> "$GITHUB_OUTPUT" + + pre-commit: + name: Pre-Commit + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + uses: ./.github/workflows/pre-commit-action.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + + docs: + name: Build Docs + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + uses: ./.github/workflows/docs-action.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + + lint: + name: Lint + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + uses: ./.github/workflows/lint-action.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + + prepare-release: + name: Prepare Release + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + needs: + - prepare-workflow + uses: ./.github/workflows/prepare-release.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + + build-source-tarball: + name: Build Source Tarball + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + needs: + - prepare-workflow + - prepare-release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Setup Python Tools Scripts + uses: ./.github/actions/setup-python-tools-scripts + + - name: Setup Salt Version + id: setup-salt-version + uses: ./.github/actions/setup-salt-version + with: + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + + - name: Build Source Tarball + uses: ./.github/actions/build-source-tarball + with: + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + + build-deps-onedir: + name: Build Dependencies Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + uses: ./.github/workflows/build-deps-onedir.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + + + build-salt-onedir: + name: Build Salt Onedir + needs: + - prepare-workflow + - build-deps-onedir + - build-source-tarball + uses: ./.github/workflows/build-salt-onedir.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + + + build-pkgs: + name: Build Salt Packages + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/build-packages.yml + with: + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + + windows-2016: + name: Windows 2016 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: windows-2016 + nox-session: ci-test-onedir + platform: windows + arch: amd64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + windows-2019: + name: Windows 2019 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: windows-2019 + nox-session: ci-test-onedir + platform: windows + arch: amd64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + windows-2022: + name: Windows 2022 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: windows-2022 + nox-session: ci-test-onedir + platform: windows + arch: amd64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + macos-12: + name: macOS 12 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action-macos.yml + with: + distro-slug: macos-12 + nox-session: ci-test-onedir + platform: darwin + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + almalinux-8: + name: Alma Linux 8 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: almalinux-8 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + almalinux-9: + name: Alma Linux 9 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: almalinux-9 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + amazonlinux-2: + name: Amazon Linux 2 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: amazonlinux-2 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + archlinux-lts: + name: Arch Linux LTS + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: archlinux-lts + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + centos-7: + name: CentOS 7 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: centos-7 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + centosstream-8: + name: CentOS Stream 8 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: centosstream-8 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + centosstream-9: + name: CentOS Stream 9 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: centosstream-9 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + debian-10: + name: Debian 10 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: debian-10 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + debian-11: + name: Debian 11 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: debian-11 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + debian-11-arm64: + name: Debian 11 Arm64 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: debian-11-arm64 + nox-session: ci-test-onedir + platform: linux + arch: aarch64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + fedora-36: + name: Fedora 36 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: fedora-36 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + opensuse-15: + name: Opensuse 15 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: opensuse-15 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + photonos-3: + name: Photon OS 3 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: photonos-3 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + photonos-4: + name: Photon OS 4 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: photonos-4 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + ubuntu-1804: + name: Ubuntu 18.04 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: ubuntu-18.04 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + ubuntu-2004: + name: Ubuntu 20.04 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: ubuntu-20.04 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + ubuntu-2004-arm64: + name: Ubuntu 20.04 Arm64 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: ubuntu-20.04-arm64 + nox-session: ci-test-onedir + platform: linux + arch: aarch64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + ubuntu-2204: + name: Ubuntu 22.04 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: ubuntu-22.04 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + ubuntu-2204-arm64: + name: Ubuntu 22.04 Arm64 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: ubuntu-22.04-arm64 + nox-session: ci-test-onedir + platform: linux + arch: aarch64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + set-pipeline-exit-status: + # This step is just so we can make github require this step, to pass checks + # on a pull request instead of requiring all + name: Set the ${{ github.workflow }} Pipeline Exit Status + if: always() + runs-on: ubuntu-latest + needs: + - prepare-workflow + - pre-commit + - docs + - lint + - build-deps-onedir + - build-salt-onedir + - build-pkgs + - windows-2016 + - windows-2019 + - windows-2022 + - macos-12 + - almalinux-8 + - almalinux-9 + - amazonlinux-2 + - archlinux-lts + - centos-7 + - centosstream-8 + - centosstream-9 + - debian-10 + - debian-11 + - debian-11-arm64 + - fedora-36 + - opensuse-15 + - photonos-3 + - photonos-4 + - ubuntu-1804 + - ubuntu-2004 + - ubuntu-2004-arm64 + - ubuntu-2204 + - ubuntu-2204-arm64 + steps: + - name: Get workflow information + id: get-workflow-info + uses: technote-space/workflow-conclusion-action@v3 + + - name: Set Pipeline Exit Status + shell: bash + run: | + if [ "${{ steps.get-workflow-info.outputs.conclusion }}" != "success" ]; then + exit 1 + else + exit 0 + fi + + - name: Done + if: always() + run: + echo "All worflows finished" diff --git a/.github/workflows/templates/ci.yml.j2 b/.github/workflows/templates/ci.yml.j2 new file mode 100644 index 00000000000..bc742b3bbbb --- /dev/null +++ b/.github/workflows/templates/ci.yml.j2 @@ -0,0 +1,512 @@ +<%- extends 'layout.yml.j2' %> + +<%- block on %> +on: + push: {} + pull_request: {} +<%- endblock on %> + +<%- block jobs %> + <{- super() }> + + pre-commit: + <%- do conclusion_needs.append('pre-commit') %> + name: Pre-Commit + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + uses: ./.github/workflows/pre-commit-action.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + + docs: + <%- do conclusion_needs.append('docs') %> + name: Build Docs + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + uses: ./.github/workflows/docs-action.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + + lint: + <%- do conclusion_needs.append('lint') %> + name: Lint + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + uses: ./.github/workflows/lint-action.yml + needs: + - prepare-workflow + with: + changed-files: ${{ needs.prepare-workflow.outputs.changed-files }} + + prepare-release: + name: Prepare Release + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + needs: + - prepare-workflow + uses: ./.github/workflows/prepare-release.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + + build-source-tarball: + name: Build Source Tarball + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + needs: + - prepare-workflow + - prepare-release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Setup Python Tools Scripts + uses: ./.github/actions/setup-python-tools-scripts + + - name: Setup Salt Version + id: setup-salt-version + uses: ./.github/actions/setup-salt-version + with: + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + + - name: Build Source Tarball + uses: ./.github/actions/build-source-tarball + with: + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + + build-deps-onedir: + <%- do conclusion_needs.append('build-deps-onedir') %> + name: Build Dependencies Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + uses: ./.github/workflows/build-deps-onedir.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + + + build-salt-onedir: + <%- do conclusion_needs.append('build-salt-onedir') %> + name: Build Salt Onedir + needs: + - prepare-workflow + - build-deps-onedir + - build-source-tarball + uses: ./.github/workflows/build-salt-onedir.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + + + build-pkgs: + <%- do conclusion_needs.append('build-pkgs') %> + name: Build Salt Packages + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/build-packages.yml + with: + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + + windows-2016: + <%- do conclusion_needs.append('windows-2016') %> + name: Windows 2016 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: windows-2016 + nox-session: ci-test-onedir + platform: windows + arch: amd64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + windows-2019: + <%- do conclusion_needs.append('windows-2019') %> + name: Windows 2019 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: windows-2019 + nox-session: ci-test-onedir + platform: windows + arch: amd64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + windows-2022: + <%- do conclusion_needs.append('windows-2022') %> + name: Windows 2022 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: windows-2022 + nox-session: ci-test-onedir + platform: windows + arch: amd64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + macos-12: + <%- do conclusion_needs.append('macos-12') %> + name: macOS 12 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['github-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action-macos.yml + with: + distro-slug: macos-12 + nox-session: ci-test-onedir + platform: darwin + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + almalinux-8: + <%- do conclusion_needs.append('almalinux-8') %> + name: Alma Linux 8 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: almalinux-8 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + almalinux-9: + <%- do conclusion_needs.append('almalinux-9') %> + name: Alma Linux 9 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: almalinux-9 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + amazonlinux-2: + <%- do conclusion_needs.append('amazonlinux-2') %> + name: Amazon Linux 2 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: amazonlinux-2 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + archlinux-lts: + <%- do conclusion_needs.append('archlinux-lts') %> + name: Arch Linux LTS + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: archlinux-lts + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + centos-7: + <%- do conclusion_needs.append('centos-7') %> + name: CentOS 7 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: centos-7 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + centosstream-8: + <%- do conclusion_needs.append('centosstream-8') %> + name: CentOS Stream 8 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: centosstream-8 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + centosstream-9: + <%- do conclusion_needs.append('centosstream-9') %> + name: CentOS Stream 9 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: centosstream-9 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + debian-10: + <%- do conclusion_needs.append('debian-10') %> + name: Debian 10 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: debian-10 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + debian-11: + <%- do conclusion_needs.append('debian-11') %> + name: Debian 11 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: debian-11 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + debian-11-arm64: + <%- do conclusion_needs.append('debian-11-arm64') %> + name: Debian 11 Arm64 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: debian-11-arm64 + nox-session: ci-test-onedir + platform: linux + arch: aarch64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + fedora-36: + <%- do conclusion_needs.append('fedora-36') %> + name: Fedora 36 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: fedora-36 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + opensuse-15: + <%- do conclusion_needs.append('opensuse-15') %> + name: Opensuse 15 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: opensuse-15 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + photonos-3: + <%- do conclusion_needs.append('photonos-3') %> + name: Photon OS 3 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: photonos-3 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + photonos-4: + <%- do conclusion_needs.append('photonos-4') %> + name: Photon OS 4 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: photonos-4 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + ubuntu-1804: + <%- do conclusion_needs.append('ubuntu-1804') %> + name: Ubuntu 18.04 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: ubuntu-18.04 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + ubuntu-2004: + <%- do conclusion_needs.append('ubuntu-2004') %> + name: Ubuntu 20.04 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: ubuntu-20.04 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + ubuntu-2004-arm64: + <%- do conclusion_needs.append('ubuntu-2004-arm64') %> + name: Ubuntu 20.04 Arm64 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: ubuntu-20.04-arm64 + nox-session: ci-test-onedir + platform: linux + arch: aarch64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + ubuntu-2204: + <%- do conclusion_needs.append('ubuntu-2204') %> + name: Ubuntu 22.04 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: ubuntu-22.04 + nox-session: ci-test-onedir + platform: linux + arch: x86_64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + + ubuntu-2204-arm64: + <%- do conclusion_needs.append('ubuntu-2204-arm64') %> + name: Ubuntu 22.04 Arm64 + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['self-hosted-runners'] }} + needs: + - prepare-workflow + - build-salt-onedir + uses: ./.github/workflows/test-action.yml + with: + distro-slug: ubuntu-22.04-arm64 + nox-session: ci-test-onedir + platform: linux + arch: aarch64 + testrun: ${{ needs.prepare-workflow.outputs.testrun }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} +<%- endblock jobs %> diff --git a/.github/workflows/templates/layout.yml.j2 b/.github/workflows/templates/layout.yml.j2 new file mode 100644 index 00000000000..106a5f1fb45 --- /dev/null +++ b/.github/workflows/templates/layout.yml.j2 @@ -0,0 +1,229 @@ +# Do not edit these workflows directly as the changes made will be overwritten. +# Instead, edit the template '<{ template }>' +--- +name: <{ workflow_name }> + +<%- block on %> + +on: + push: {} + pull_request: {} + +<%- endblock on %> + +<%- block env %> + +env: + COLUMNS: 160 + CACHE_SEED: SEED-4 # Bump the number to invalidate all caches + RELENV_DATA: "${{ github.workspace }}/.relenv" + +<%- endblock env %> + +<%- block permissions %> + +permissions: + contents: read # for dorny/paths-filter to fetch a list of changed files + pull-requests: read # for dorny/paths-filter to read pull requests + +<%- endblock permissions %> + +<%- block concurrency %> + +concurrency: + # Concurrency is defined in a way that concurrent builds against branches do + # not cancel previous builds. + # However, for every new build against the same pull request source branch, + # all older builds against that same branch get canceled. + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +<%- endblock concurrency %> + +<%- block jobs %> + +jobs: + + <%- block prepare_workflow_job %> + <%- do conclusion_needs.append('prepare-workflow') %> + prepare-workflow: + name: Prepare Workflow Run + runs-on: ubuntu-latest + outputs: + jobs: ${{ steps.define-jobs.outputs.jobs }} + changed-files: ${{ steps.process-changed-files.outputs.changed-files }} + testrun: ${{ steps.define-testrun.outputs.testrun }} + salt-version: ${{ steps.setup-salt-version.outputs.salt-version }} + cache-seed: ${{ steps.set-cache-seed.outputs.cache-seed }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Full clone to also get the tags to get the right salt version + + - name: Get Changed Files + if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}} + id: changed-files + uses: dorny/paths-filter@v2 + with: + token: ${{ github.token }} + list-files: json + filters: | + repo: + - added|modified: + - '**' + doc-requirements: + - added|modified: &doc_requirements + - requirements/static/ci/py3.*/docs.txt + lint-requirements: + - added|modified: &lint_requirements + - requirements/static/ci/py3.*/lint.txt + pkg_requirements: + - added|modified: &pkg_requirements + - requirements/static/pkg/py3.*/darwin.txt + - requirements/static/pkg/py3.*/linux.txt + - requirements/static/pkg/py3.*/freebsd.txt + - requirements/static/pkg/py3.*/windows.txt + test_requirements: + - added|modified: &test_requirements + - requirements/static/ci/py3.*/darwin.txt + - requirements/static/ci/py3.*/linux.txt + - requirements/static/ci/py3.*/freebsd.txt + - requirements/static/ci/py3.*/windows.txt + - requirements/static/ci/py3.*/darwin-crypto.txt + - requirements/static/ci/py3.*/linux-crypto.txt + - requirements/static/ci/py3.*/freebsd-crypto.txt + - requirements/static/ci/py3.*/windows-crypto.txt + deleted: + - deleted: + - '**' + docs: + - added|modified: + - doc/** + - *doc_requirements + salt: + - added|modified: &salt_added_modified + - setup.py + - noxfile.py + - salt/**/*.py + - tasks/**/*.py + - tools/**/*.py + tests: + - added|modified: &tests_added_modified + - tests/**/*.py + lint: + - added|modified: + - .pylintrc + - *lint_requirements + golden_images: + - added|modified: + - cicd/golden-images.json + testrun: + - added|modified: + - *salt_added_modified + - *tests_added_modified + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Setup Python Tools Scripts + uses: ./.github/actions/setup-python-tools-scripts + + - name: Pretty Print The GH Actions Event + run: + tools ci print-gh-event + + - name: Setup Salt Version + id: setup-salt-version + uses: ./.github/actions/setup-salt-version + with: + salt-version: "" + + - name: Write Changed Files To A Local File + if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}} + run: + echo '${{ toJSON(steps.changed-files.outputs) }}' > changed-files.json + + - name: Check Local Changed Files Contents + if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}} + run: + cat changed-files.json + + - name: Process Changed Files + if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}} + id: process-changed-files + run: + tools ci process-changed-files ${{ github.event_name }} changed-files.json + + - name: Check Collected Changed Files + if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}} + run: + echo '${{ steps.process-changed-files.outputs.changed-files }}' | jq -C '.' + + - name: Define Jobs To Run + id: define-jobs + run: + tools ci define-jobs ${{ github.event_name }} + + - name: Check Collected Jobs + run: + echo '${{ steps.define-jobs.outputs.jobs }}' | jq -C '.' + + - name: Define Testrun + id: define-testrun + run: + tools ci define-testrun ${{ github.event_name }} changed-files.json + + - name: Check Defined Test Run + run: + echo '${{ steps.define-testrun.outputs.testrun }}' | jq -C '.' + + - name: Check Contents of generated testrun-changed-files.txt + if: ${{ fromJSON(steps.define-testrun.outputs.testrun)['type'] != 'full' }} + run: + cat testrun-changed-files.txt || true + + - name: Upload testrun-changed-files.txt + if: ${{ fromJSON(steps.define-testrun.outputs.testrun)['type'] != 'full' }} + uses: actions/upload-artifact@v3 + with: + name: testrun-changed-files.txt + path: testrun-changed-files.txt + + - name: Set Cache Seed Output + id: set-cache-seed + run: | + echo "cache-seed=${{ env.CACHE_SEED }}" >> "$GITHUB_OUTPUT" + <%- endblock prepare_workflow_job %> + +<%- endblock jobs %> + + set-pipeline-exit-status: + # This step is just so we can make github require this step, to pass checks + # on a pull request instead of requiring all + name: Set the ${{ github.workflow }} Pipeline Exit Status + if: always() + runs-on: ubuntu-latest + needs: + <%- for need in conclusion_needs %> + - <{ need }> + <%- endfor %> + steps: + - name: Get workflow information + id: get-workflow-info + uses: technote-space/workflow-conclusion-action@v3 + + - name: Set Pipeline Exit Status + shell: bash + run: | + if [ "${{ steps.get-workflow-info.outputs.conclusion }}" != "success" ]; then + exit 1 + else + exit 0 + fi + + - name: Done + if: always() + run: + echo "All worflows finished" diff --git a/.github/workflows/templates/scheduled.yml.j2 b/.github/workflows/templates/scheduled.yml.j2 new file mode 100644 index 00000000000..2a7e7407cc4 --- /dev/null +++ b/.github/workflows/templates/scheduled.yml.j2 @@ -0,0 +1,16 @@ +<%- extends 'ci.yml.j2' %> + +<%- block on %> + +on: + schedule: + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onschedule + - cron: '0 */8 * * *' # Run every 8 hours +<%- endblock %> + +<%- block concurrency %> + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: false +<%- endblock %> diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3620cebf885..290dd90268f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,11 +3,28 @@ default_language_version: exclude: ^(doc/_static/.*|doc/_themes/.*)$ repos: + - repo: https://github.com/rhysd/actionlint rev: v1.6.23 hooks: - id: actionlint + - repo: https://github.com/s0undt3ch/python-tools-scripts + rev: "0.10.1" + hooks: + - id: tools + alias: generate-workflows + name: Generate GitHub Workflow Templates + files: ^.github/workflows/templates/.*$ + pass_filenames: false + args: + - pre-commit + - generate-workflows + additional_dependencies: + - boto3==1.21.46 + - pyyaml==6.0 + - jinja2==3.1.2 + - repo: https://github.com/saltstack/pip-tools-compile-impersonate rev: "4.6" hooks: diff --git a/requirements/static/ci/py3.10/tools.txt b/requirements/static/ci/py3.10/tools.txt index 858e534198c..a18b1fa9016 100644 --- a/requirements/static/ci/py3.10/tools.txt +++ b/requirements/static/ci/py3.10/tools.txt @@ -16,10 +16,14 @@ botocore==1.24.46 # s3transfer commonmark==0.9.1 # via rich +jinja2==3.1.2 + # via -r requirements/static/ci/tools.in jmespath==1.0.1 # via # boto3 # botocore +markupsafe==2.1.2 + # via jinja2 pygments==2.13.0 # via rich python-dateutil==2.8.2 diff --git a/requirements/static/ci/py3.9/tools.txt b/requirements/static/ci/py3.9/tools.txt index 7226647a7e2..347617a251e 100644 --- a/requirements/static/ci/py3.9/tools.txt +++ b/requirements/static/ci/py3.9/tools.txt @@ -16,10 +16,14 @@ botocore==1.24.46 # s3transfer commonmark==0.9.1 # via rich +jinja2==3.1.2 + # via -r requirements/static/ci/tools.in jmespath==1.0.1 # via # boto3 # botocore +markupsafe==2.1.2 + # via jinja2 pygments==2.13.0 # via rich python-dateutil==2.8.2 diff --git a/requirements/static/ci/tools.in b/requirements/static/ci/tools.in index 08ccf6d7245..565c1c98b21 100644 --- a/requirements/static/ci/tools.in +++ b/requirements/static/ci/tools.in @@ -2,3 +2,4 @@ python-tools-scripts >= 0.10.1 attrs boto3 pyyaml +jinja2 diff --git a/tools/__init__.py b/tools/__init__.py index 9080846a478..4180c4cd8ac 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -4,6 +4,7 @@ import tools.changelog import tools.ci import tools.docs import tools.pkg +import tools.pre_commit import tools.vm for name in ("boto3", "botocore", "urllib3"): diff --git a/tools/pre_commit.py b/tools/pre_commit.py new file mode 100644 index 00000000000..c3f63b521d6 --- /dev/null +++ b/tools/pre_commit.py @@ -0,0 +1,71 @@ +""" +These commands are used by pre-commit. +""" +# pylint: disable=resource-leakage,broad-except +from __future__ import annotations + +import logging +import pathlib + +from jinja2 import Environment, FileSystemLoader +from ptscripts import Context, command_group + +log = logging.getLogger(__name__) + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +WORKFLOWS = REPO_ROOT / ".github" / "workflows" +TEMPLATES = WORKFLOWS / "templates" + +# Define the command group +cgroup = command_group( + name="pre-commit", help="Pre-Commit Related Commands", description=__doc__ +) + + +class NoDuplicatesList(list): + def append(self, need): + if need not in self: + super().append(need) + + +@cgroup.command( + name="generate-workflows", +) +def generate_workflows(ctx: Context): + """ + Generate GitHub Actions Workflows + """ + workflows = { + "CI": { + "template": "ci.yml", + }, + "Scheduled": { + "template": "scheduled.yml", + }, + } + env = Environment( + block_start_string="<%", + block_end_string="%>", + variable_start_string="<{", + variable_end_string="}>", + extensions=[ + "jinja2.ext.do", + ], + loader=FileSystemLoader(str(TEMPLATES)), + ) + for workflow_name, details in workflows.items(): + template = details["template"] + workflow_path = WORKFLOWS / template + template_path = TEMPLATES / f"{template}.j2" + ctx.info( + f"Generating '{workflow_path.relative_to(REPO_ROOT)}' from " + f"template '{template_path.relative_to(REPO_ROOT)}' ..." + ) + context = { + "template": template_path.relative_to(REPO_ROOT), + "workflow_name": workflow_name, + "conclusion_needs": NoDuplicatesList(), + } + loaded_template = env.get_template(f"{template}.j2") + rendered_template = loaded_template.render(**context) + workflow_path.write_text(rendered_template.rstrip() + "\n")