From 7de3f96ee51b685e2f92c411b4d9763692da335b Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 18 Mar 2024 17:58:44 +0000 Subject: [PATCH 1/5] Remove invalid properties from existing GH actions --- .github/actions/build-onedir-deps/action.yml | 15 +++------------ .github/actions/build-onedir-salt/action.yml | 16 +++------------- .github/actions/build-source-tarball/action.yml | 9 +-------- .github/actions/cached-virtualenv/action.yml | 11 ++++------- .github/actions/download-artifact/action.yml | 1 + .github/actions/get-python-version/action.yml | 7 ++++++- .github/actions/setup-actionlint/action.yml | 3 ++- .github/actions/setup-pre-commit/action.yml | 6 ------ .../setup-python-tools-scripts/action.yml | 8 +------- .github/actions/setup-relenv/action.yml | 12 ++---------- .github/actions/setup-salt-version/action.yml | 13 +++++-------- .github/actions/setup-shellcheck/action.yml | 3 ++- .github/actions/upload-artifact/action.yml | 1 + 13 files changed, 31 insertions(+), 74 deletions(-) diff --git a/.github/actions/build-onedir-deps/action.yml b/.github/actions/build-onedir-deps/action.yml index 511fe5a5275..cdfdf7d917a 100644 --- a/.github/actions/build-onedir-deps/action.yml +++ b/.github/actions/build-onedir-deps/action.yml @@ -1,37 +1,26 @@ --- name: build-onedir-deps description: Build Onedir Dependencies + inputs: platform: required: true - type: string description: The platform to build arch: required: true - type: string description: The platform arch to build python-version: required: true - type: string description: The python version to build package-name: required: false - type: string description: The onedir package name to create default: salt cache-prefix: required: true - type: string description: Seed used to invalidate caches -env: - COLUMNS: 190 - PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ - PIP_EXTRA_INDEX_URL: https://pypi.org/simple - RELENV_BUILDENV: 1 - - runs: using: composite @@ -56,6 +45,8 @@ runs: - name: Install Salt Onedir Package Dependencies shell: bash if: steps.onedir-pkg-cache.outputs.cache-hit != 'true' + env: + RELENV_BUILDENV: "1" run: | tools pkg build onedir-dependencies --arch ${{ inputs.arch }} --python-version ${{ inputs.python-version }} --package-name artifacts/${{ inputs.package-name }} --platform ${{ inputs.platform }} diff --git a/.github/actions/build-onedir-salt/action.yml b/.github/actions/build-onedir-salt/action.yml index 6952ccf98e4..2534cec01ed 100644 --- a/.github/actions/build-onedir-salt/action.yml +++ b/.github/actions/build-onedir-salt/action.yml @@ -1,41 +1,29 @@ --- name: build-onedir-salt description: Build Onedir Package + inputs: platform: required: true - type: string description: The platform to build arch: required: true - type: string description: The platform arch to build package-name: required: false - type: string description: The onedir package name to create default: salt cache-prefix: required: true - type: string description: Seed used to invalidate caches python-version: required: true - type: string description: The python version to build salt-version: - type: string required: true description: The Salt version to set prior to building packages. -env: - COLUMNS: 190 - PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ - PIP_EXTRA_INDEX_URL: https://pypi.org/simple - RELENV_BUILDENV: 1 - - runs: using: composite @@ -64,6 +52,8 @@ runs: - name: Install Salt Into Onedir shell: bash + env: + RELENV_BUILDENV: "1" run: | tools pkg build salt-onedir salt-${{ inputs.salt-version }}.tar.gz --platform ${{ inputs.platform }} --package-name artifacts/${{ inputs.package-name }} diff --git a/.github/actions/build-source-tarball/action.yml b/.github/actions/build-source-tarball/action.yml index 8c662d8ee8b..22a5a674fca 100644 --- a/.github/actions/build-source-tarball/action.yml +++ b/.github/actions/build-source-tarball/action.yml @@ -1,24 +1,17 @@ --- name: build-source-tarball description: Build Source Tarball + inputs: salt-version: - type: string required: true description: The Salt version to set prior to building the tarball. nox-version: required: false - type: string description: The version of Nox to install default: "2022.8.7" -env: - COLUMNS: 190 - PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ - PIP_EXTRA_INDEX_URL: https://pypi.org/simple - - runs: using: composite diff --git a/.github/actions/cached-virtualenv/action.yml b/.github/actions/cached-virtualenv/action.yml index f135d9116e5..4d0ddf099fa 100644 --- a/.github/actions/cached-virtualenv/action.yml +++ b/.github/actions/cached-virtualenv/action.yml @@ -5,26 +5,23 @@ description: Setup a cached python virtual environment inputs: name: required: true - type: string description: The Virtualenv Name cache-seed: required: true - type: string description: Seed used to invalidate caches + outputs: cache-hit: + description: 'A boolean value to indicate an exact match was found for the primary key' value: ${{ steps.cache-virtualenv.outputs.cache-hit }} cache-key: + description: The value of the cache key value: ${{ steps.setup-cache-key.outputs.cache-key }} python-executable: + description: The path to the virtualenv's python executable value: ${{ steps.define-python-executable.outputs.python-executable }} -env: - PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ - PIP_EXTRA_INDEX_URL: https://pypi.org/simple - - runs: using: composite diff --git a/.github/actions/download-artifact/action.yml b/.github/actions/download-artifact/action.yml index 12a097f517f..f1b8e547b14 100644 --- a/.github/actions/download-artifact/action.yml +++ b/.github/actions/download-artifact/action.yml @@ -20,6 +20,7 @@ inputs: without overriding the existing archives. required: false + runs: using: composite steps: diff --git a/.github/actions/get-python-version/action.yml b/.github/actions/get-python-version/action.yml index f2b045f7ca7..a5bdc7f32a2 100644 --- a/.github/actions/get-python-version/action.yml +++ b/.github/actions/get-python-version/action.yml @@ -1,19 +1,24 @@ --- name: get-python-version description: Setup Relenv + inputs: python-binary: required: true - type: string description: The python binary to get the version from + outputs: binary: + description: The python binary executable value: ${{ steps.get-python-version.outputs.binary }} version: + description: The python version value: ${{ steps.get-python-version.outputs.version }} full-version: + description: The full python version value: ${{ steps.get-python-version.outputs.full-version }} version-sha256sum: + description: The sha256sum of the version value: ${{ steps.get-python-version.outputs.version-sha256sum }} diff --git a/.github/actions/setup-actionlint/action.yml b/.github/actions/setup-actionlint/action.yml index f1a81aaf35f..e965a9c60cf 100644 --- a/.github/actions/setup-actionlint/action.yml +++ b/.github/actions/setup-actionlint/action.yml @@ -1,15 +1,16 @@ --- name: setup-actionlint description: Setup actionlint + inputs: version: description: The version of actionlint default: 1.6.26 cache-seed: required: true - type: string description: Seed used to invalidate caches + runs: using: composite steps: diff --git a/.github/actions/setup-pre-commit/action.yml b/.github/actions/setup-pre-commit/action.yml index 82b8eef583d..ae62fa51edb 100644 --- a/.github/actions/setup-pre-commit/action.yml +++ b/.github/actions/setup-pre-commit/action.yml @@ -4,19 +4,13 @@ description: Setup 'pre-commit' inputs: version: - type: string description: Pre-commit version to install required: true default: 3.0.3 cache-seed: required: true - type: string description: Seed used to invalidate caches -env: - PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ - PIP_EXTRA_INDEX_URL: https://pypi.org/simple - runs: using: composite diff --git a/.github/actions/setup-python-tools-scripts/action.yml b/.github/actions/setup-python-tools-scripts/action.yml index 9d5ff710346..d060c9d4a1b 100644 --- a/.github/actions/setup-python-tools-scripts/action.yml +++ b/.github/actions/setup-python-tools-scripts/action.yml @@ -5,23 +5,17 @@ description: Setup 'python-tools-scripts' inputs: cache-prefix: required: true - type: string description: Seed used to invalidate caches cwd: - type: string description: The directory the salt checkout is located in default: "." outputs: version: + description: "Return the python-tools-scripts version" value: ${{ steps.get-version.outputs.version }} -env: - PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ - PIP_EXTRA_INDEX_URL: https://pypi.org/simple - - runs: using: composite diff --git a/.github/actions/setup-relenv/action.yml b/.github/actions/setup-relenv/action.yml index c4cfd33f545..d8303a92447 100644 --- a/.github/actions/setup-relenv/action.yml +++ b/.github/actions/setup-relenv/action.yml @@ -1,39 +1,31 @@ --- name: setup-relenv description: Setup Relenv + inputs: platform: required: true - type: string description: The platform to build arch: required: true - type: string description: The platform arch to build python-version: required: true - type: string description: The version of python to build cache-seed: required: true - type: string description: Seed used to invalidate caches version: required: false - type: string description: The version of relenv to use default: 0.13.2 outputs: version: + description: The relenv version value: ${{ inputs.version }} -env: - PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ - PIP_EXTRA_INDEX_URL: https://pypi.org/simple - - runs: using: composite diff --git a/.github/actions/setup-salt-version/action.yml b/.github/actions/setup-salt-version/action.yml index 9f21d6f002b..7d54c1c95b5 100644 --- a/.github/actions/setup-salt-version/action.yml +++ b/.github/actions/setup-salt-version/action.yml @@ -1,32 +1,29 @@ --- name: setup-salt-version description: Setup Salt Version + inputs: cwd: - type: string default: "" + description: The current working directory to use salt-version: - type: string default: "" description: > The Salt version to set prior to running tests or building packages. If not set, it is discover at run time, like, for example, capturing the output of running `python3 salt/version.py` validate-version: - type: boolean - default: false + default: "false" description: Validate the passed version. release: - type: boolean - default: false + default: "false" description: This is a release of salt. + outputs: salt-version: value: ${{ steps.setup-salt-version.outputs.salt-version }} description: The Salt version written to `salt/_version.txt` -env: - COLUMNS: 190 runs: using: composite diff --git a/.github/actions/setup-shellcheck/action.yml b/.github/actions/setup-shellcheck/action.yml index 2c86c98a072..9e5ad30365f 100644 --- a/.github/actions/setup-shellcheck/action.yml +++ b/.github/actions/setup-shellcheck/action.yml @@ -1,15 +1,16 @@ --- name: setup-shellcheck description: Setup shellcheck + inputs: version: description: The version of shellcheck default: v0.9.0 cache-seed: required: true - type: string description: Seed used to invalidate caches + runs: using: composite steps: diff --git a/.github/actions/upload-artifact/action.yml b/.github/actions/upload-artifact/action.yml index 6c9e940a3b3..eb00196a339 100644 --- a/.github/actions/upload-artifact/action.yml +++ b/.github/actions/upload-artifact/action.yml @@ -37,6 +37,7 @@ inputs: without overriding the existing archives. required: false + runs: using: composite steps: From f7d7a50a848bd92b2e6ac0fbd74fb2570140f633 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 18 Mar 2024 18:06:26 +0000 Subject: [PATCH 2/5] Force cache invalidation by setting cache seed to 1 --- .github/workflows/ci.yml | 2 +- .github/workflows/nightly.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/scheduled.yml | 2 +- .github/workflows/staging.yml | 2 +- .github/workflows/templates/layout.yml.jinja | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c89728df8a3..cbdad9d3cd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ on: env: COLUMNS: 190 - CACHE_SEED: SEED-4 # Bump the number to invalidate all caches + CACHE_SEED: SEED-1 # Bump the number to invalidate all caches RELENV_DATA: "${{ github.workspace }}/.relenv" PIP_DISABLE_PIP_VERSION_CHECK: "1" RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b4be3f81e05..6b721033fd7 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -22,7 +22,7 @@ on: env: COLUMNS: 190 - CACHE_SEED: SEED-4 # Bump the number to invalidate all caches + CACHE_SEED: SEED-1 # Bump the number to invalidate all caches RELENV_DATA: "${{ github.workspace }}/.relenv" PIP_DISABLE_PIP_VERSION_CHECK: "1" RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d7f4d158055..9bf81b4f897 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ on: env: COLUMNS: 190 - CACHE_SEED: SEED-4 # Bump the number to invalidate all caches + CACHE_SEED: SEED-1 # Bump the number to invalidate all caches RELENV_DATA: "${{ github.workspace }}/.relenv" PIP_DISABLE_PIP_VERSION_CHECK: "1" RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 724b529fb74..e39b9bec08d 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -12,7 +12,7 @@ on: env: COLUMNS: 190 - CACHE_SEED: SEED-4 # Bump the number to invalidate all caches + CACHE_SEED: SEED-1 # Bump the number to invalidate all caches RELENV_DATA: "${{ github.workspace }}/.relenv" PIP_DISABLE_PIP_VERSION_CHECK: "1" RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index c29e4877dac..159e9ca669b 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -37,7 +37,7 @@ on: env: COLUMNS: 190 - CACHE_SEED: SEED-4 # Bump the number to invalidate all caches + CACHE_SEED: SEED-1 # Bump the number to invalidate all caches RELENV_DATA: "${{ github.workspace }}/.relenv" PIP_DISABLE_PIP_VERSION_CHECK: "1" RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" diff --git a/.github/workflows/templates/layout.yml.jinja b/.github/workflows/templates/layout.yml.jinja index 31ddc11e1a0..a74d71c22c4 100644 --- a/.github/workflows/templates/layout.yml.jinja +++ b/.github/workflows/templates/layout.yml.jinja @@ -34,7 +34,7 @@ on: env: COLUMNS: 190 - CACHE_SEED: SEED-4 # Bump the number to invalidate all caches + CACHE_SEED: SEED-1 # Bump the number to invalidate all caches RELENV_DATA: "${{ github.workspace }}/.relenv" PIP_DISABLE_PIP_VERSION_CHECK: "1" RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" From b1162dfaa3d5c7f38396912b8fffbef40a3c4818 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 18 Mar 2024 15:49:13 +0000 Subject: [PATCH 3/5] Add our own GH cache action This actions wraps actions/cache and forces an early exit if 'fail-on-cache-miss' is true Based on the work from https://github.com/saltstack/salt/pull/66240 --- .github/actions/build-onedir-deps/action.yml | 2 +- .github/actions/build-onedir-salt/action.yml | 2 +- .github/actions/cache/action.yml | 77 +++++++++++++++++++ .github/actions/cached-virtualenv/action.yml | 2 +- .github/actions/setup-actionlint/action.yml | 2 +- .github/actions/setup-pre-commit/action.yml | 2 +- .../setup-python-tools-scripts/action.yml | 2 +- .github/actions/setup-relenv/action.yml | 2 +- .github/actions/setup-shellcheck/action.yml | 2 +- .github/workflows/build-deps-ci-action.yml | 6 +- .github/workflows/build-docs.yml | 2 +- .../templates/build-deps-ci-action.yml.jinja | 6 +- 12 files changed, 92 insertions(+), 15 deletions(-) create mode 100644 .github/actions/cache/action.yml diff --git a/.github/actions/build-onedir-deps/action.yml b/.github/actions/build-onedir-deps/action.yml index cdfdf7d917a..c34133bc369 100644 --- a/.github/actions/build-onedir-deps/action.yml +++ b/.github/actions/build-onedir-deps/action.yml @@ -28,7 +28,7 @@ runs: - name: Cache Deps Onedir Package Directory id: onedir-pkg-cache - uses: actions/cache@v3.3.1 + uses: ./.github/actions/cache with: path: artifacts/${{ inputs.package-name }} key: > diff --git a/.github/actions/build-onedir-salt/action.yml b/.github/actions/build-onedir-salt/action.yml index 2534cec01ed..0e3888cada6 100644 --- a/.github/actions/build-onedir-salt/action.yml +++ b/.github/actions/build-onedir-salt/action.yml @@ -31,7 +31,7 @@ runs: - name: Download Cached Deps Onedir Package Directory id: onedir-bare-cache - uses: actions/cache@v3.3.1 + uses: ./.github/actions/cache with: path: artifacts/${{ inputs.package-name }} key: > diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml new file mode 100644 index 00000000000..ae416246d79 --- /dev/null +++ b/.github/actions/cache/action.yml @@ -0,0 +1,77 @@ +--- +name: cache +description: GitHub Actions Cache +inputs: + path: + description: 'A list of files, directories, and wildcard patterns to cache and restore' + required: true + key: + description: 'An explicit key for restoring and saving the cache' + required: true + restore-keys: + description: 'An ordered list of keys to use for restoring stale cache if no cache hit occurred for key. Note `cache-hit` returns false in this case.' + required: false + upload-chunk-size: + description: 'The chunk size used to split up large files during upload, in bytes' + required: false + enableCrossOsArchive: + description: 'An optional boolean when enabled, allows windows runners to save or restore caches that can be restored or saved respectively on other platforms' + default: 'false' + required: false + fail-on-cache-miss: + description: 'Fail the workflow if cache entry is not found' + default: 'false' + required: false + lookup-only: + description: 'Check if a cache entry exists for the given input(s) (key, restore-keys) without downloading the cache' + default: 'false' + required: false + save-always: + description: 'Run the post step to save the cache even if another step before fails' + default: 'false' + required: false + +outputs: + cache-hit: + description: 'A boolean value to indicate an exact match was found for the primary key' + value: ${{ steps.github-cache.outputs.cache-hit }} + +runs: + using: composite + + steps: + + - name: Map inputs to environment variables + shell: bash + run: | + echo "GHA_CACHE_PATH=${{ inputs.path }}" | tee -a "${GITHUB_ENV}" + echo "GHA_CACHE_KEY=${{ inputs.key }}" | tee -a "${GITHUB_ENV}" + echo "GHA_CACHE_ENABLE_CROSS_OS_ARCHIVE=${{ inputs.enableCrossOsArchive }}" | tee -a "${GITHUB_ENV}" + echo "GHA_CACHE_FAIL_ON_CACHE_MISS=${{ inputs.fail-on-cache-miss }}" | tee -a "${GITHUB_ENV}" + echo "GHA_CACHE_LOOKUP_ONLY=${{ inputs.lookup-only }}" | tee -a "${GITHUB_ENV}" + echo "GHA_CACHE_SAVE_ALWAYS=${{ inputs.save-always }}" | tee -a "${GITHUB_ENV}" + echo "GHA_CACHE_RESTORE_KEYS=${{ inputs.restore-keys }}" | tee -a "${GITHUB_ENV}" + echo "GHA_CACHE_UPLOAD_CHUNK_SIZE=${{ inputs.upload-chunk-size }}" | tee -a "${GITHUB_ENV}" + + - name: Cache Provided Path (GitHub Actions) + id: github-cache + uses: actions/cache@v4 + with: + path: ${{ env.GHA_CACHE_PATH }} + key: ${{ env.GHA_CACHE_KEY }} + enableCrossOsArchive: ${{ env.GHA_CACHE_ENABLE_CROSS_OS_ARCHIVE }} + fail-on-cache-miss: ${{ env.GHA_CACHE_FAIL_ON_CACHE_MISS }} + lookup-only: ${{ env.GHA_CACHE_LOOKUP_ONLY }} + save-always: ${{ env.GHA_CACHE_SAVE_ALWAYS }} + restore-keys: ${{ env.GHA_CACHE_RESTORE_KEYS }} + upload-chunk-size: ${{ env.GHA_CACHE_UPLOAD_CHUNK_SIZE }} + + - name: Verify 'fail-on-cache-miss' + if: ${{ inputs.fail-on-cache-miss == 'true' }} + shell: bash + run: | + CACHE_HIT="${{ steps.github-cache.outputs.cache-hit }}" + if [ "$CACHE_HIT" != "true" ]; then + echo "No cache hit and fail-on-cache-miss is set to true." + exit 1 + fi diff --git a/.github/actions/cached-virtualenv/action.yml b/.github/actions/cached-virtualenv/action.yml index 4d0ddf099fa..a24b805599d 100644 --- a/.github/actions/cached-virtualenv/action.yml +++ b/.github/actions/cached-virtualenv/action.yml @@ -51,7 +51,7 @@ runs: - name: Cache VirtualEnv id: cache-virtualenv - uses: actions/cache@v3.3.1 + uses: ./.github/actions/cache with: key: ${{ steps.setup-cache-key.outputs.cache-key }} path: ${{ steps.virtualenv-path.outputs.venv-path }} diff --git a/.github/actions/setup-actionlint/action.yml b/.github/actions/setup-actionlint/action.yml index e965a9c60cf..44b68e86c9f 100644 --- a/.github/actions/setup-actionlint/action.yml +++ b/.github/actions/setup-actionlint/action.yml @@ -16,7 +16,7 @@ runs: steps: - name: Cache actionlint Binary - uses: actions/cache@v3.3.1 + uses: ./.github/actions/cache with: path: /usr/local/bin/actionlint key: ${{ inputs.cache-seed }}|${{ runner.os }}|${{ runner.arch }}|actionlint|${{ inputs.version }} diff --git a/.github/actions/setup-pre-commit/action.yml b/.github/actions/setup-pre-commit/action.yml index ae62fa51edb..2acd58895c5 100644 --- a/.github/actions/setup-pre-commit/action.yml +++ b/.github/actions/setup-pre-commit/action.yml @@ -30,7 +30,7 @@ runs: ${{ steps.pre-commit-virtualenv.outputs.python-executable }} -m pip install pre-commit==${{ inputs.version }} - name: Cache Pre-Commit Hooks - uses: actions/cache@v3.3.1 + uses: ./.github/actions/cache id: pre-commit-hooks-cache with: key: ${{ steps.pre-commit-virtualenv.outputs.cache-key }}|${{ inputs.version }}|${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/.github/actions/setup-python-tools-scripts/action.yml b/.github/actions/setup-python-tools-scripts/action.yml index d060c9d4a1b..e640ffe86f7 100644 --- a/.github/actions/setup-python-tools-scripts/action.yml +++ b/.github/actions/setup-python-tools-scripts/action.yml @@ -44,7 +44,7 @@ runs: cache-seed: tools|${{ steps.venv-hash.outputs.venv-hash }} - name: Restore Python Tools Virtualenvs Cache - uses: actions/cache@v3.3.1 + uses: ./.github/actions/cache with: path: ${{ inputs.cwd }}/.tools-venvs key: ${{ inputs.cache-prefix }}|${{ steps.venv-hash.outputs.venv-hash }} diff --git a/.github/actions/setup-relenv/action.yml b/.github/actions/setup-relenv/action.yml index d8303a92447..825d0401d12 100644 --- a/.github/actions/setup-relenv/action.yml +++ b/.github/actions/setup-relenv/action.yml @@ -37,7 +37,7 @@ runs: python3 -m pip install relenv==${{ inputs.version }} - name: Cache Relenv Data Directory - uses: actions/cache@v3.3.1 + uses: ./.github/actions/cache with: path: ${{ github.workspace }}/.relenv key: ${{ inputs.cache-seed }}|relenv|${{ inputs.version }}|${{ inputs.python-version }}|${{ inputs.platform }}|${{ inputs.arch }} diff --git a/.github/actions/setup-shellcheck/action.yml b/.github/actions/setup-shellcheck/action.yml index 9e5ad30365f..e6d0697933d 100644 --- a/.github/actions/setup-shellcheck/action.yml +++ b/.github/actions/setup-shellcheck/action.yml @@ -16,7 +16,7 @@ runs: steps: - name: Cache shellcheck Binary - uses: actions/cache@v3.3.1 + uses: ./.github/actions/cache with: path: /usr/local/bin/shellcheck key: ${{ inputs.cache-seed }}|${{ runner.os }}|${{ runner.arch }}|shellcheck|${{ inputs.version }} diff --git a/.github/workflows/build-deps-ci-action.yml b/.github/workflows/build-deps-ci-action.yml index 82389dbb448..26ad6fa8b65 100644 --- a/.github/workflows/build-deps-ci-action.yml +++ b/.github/workflows/build-deps-ci-action.yml @@ -74,7 +74,7 @@ jobs: - name: Cache nox.linux.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }} id: nox-dependencies-cache - uses: actions/cache@v3.3.1 + uses: ./.github/actions/cache with: path: nox.linux.${{ matrix.arch }}.tar.* key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|linux|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }} @@ -183,7 +183,7 @@ jobs: - name: Cache nox.macos.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }} id: nox-dependencies-cache - uses: actions/cache@v3.3.1 + uses: ./.github/actions/cache with: path: nox.macos.${{ matrix.arch }}.tar.* key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|macos|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }} @@ -272,7 +272,7 @@ jobs: - name: Cache nox.windows.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }} id: nox-dependencies-cache - uses: actions/cache@v3.3.1 + uses: ./.github/actions/cache with: path: nox.windows.${{ matrix.arch }}.tar.* key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|windows|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }} diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 415cdfc7d19..64c19ccb8a5 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -69,7 +69,7 @@ jobs: - name: Cache Python Tools Docs Virtualenv id: tools-venvs-dependencies-cache - uses: actions/cache@v3.3.1 + uses: ./.github/actions/cache with: path: .tools-venvs/docs key: ${{ inputs.cache-seed }}|${{ github.workflow }}|${{ github.job }}|tools-venvs|${{ steps.python-tools-scripts.outputs.version }}|docs|${{ steps.get-python-version.outputs.version }}|${{ hashFiles('requirements/**/docs.txt') }} diff --git a/.github/workflows/templates/build-deps-ci-action.yml.jinja b/.github/workflows/templates/build-deps-ci-action.yml.jinja index a43f7b43d01..1038d2b71dd 100644 --- a/.github/workflows/templates/build-deps-ci-action.yml.jinja +++ b/.github/workflows/templates/build-deps-ci-action.yml.jinja @@ -74,7 +74,7 @@ jobs: - name: Cache nox.linux.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }} id: nox-dependencies-cache - uses: actions/cache@v3.3.1 + uses: ./.github/actions/cache with: path: nox.linux.${{ matrix.arch }}.tar.* key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|linux|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }} @@ -183,7 +183,7 @@ jobs: - name: Cache nox.macos.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }} id: nox-dependencies-cache - uses: actions/cache@v3.3.1 + uses: ./.github/actions/cache with: path: nox.macos.${{ matrix.arch }}.tar.* key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|macos|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }} @@ -274,7 +274,7 @@ jobs: - name: Cache nox.windows.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }} id: nox-dependencies-cache - uses: actions/cache@v3.3.1 + uses: ./.github/actions/cache with: path: nox.windows.${{ matrix.arch }}.tar.* key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|windows|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }} From 311652331d993a94a13de43ffe73d716b423bfa5 Mon Sep 17 00:00:00 2001 From: Felippe Burk Date: Mon, 18 Mar 2024 21:47:51 +0000 Subject: [PATCH 4/5] Initial commit of s3 caching --- .github/actions/build-onedir-deps/action.yml | 1 - .github/actions/cache/action.yml | 28 +++++++++++++++++-- .github/workflows/build-deps-ci-action.yml | 4 +++ .github/workflows/build-deps-onedir.yml | 7 ++++- .github/workflows/build-salt-onedir.yml | 2 ++ .github/workflows/nightly.yml | 12 ++++++++ .github/workflows/release.yml | 13 ++++++++- .github/workflows/staging.yml | 12 ++++++++ .../templates/build-deps-ci-action.yml.jinja | 4 +++ .../workflows/templates/build-repos.yml.jinja | 2 ++ .github/workflows/templates/release.yml.jinja | 15 +++++++++- .../test-package-downloads-action.yml.jinja | 6 ++++ .../test-package-downloads-action.yml | 6 ++++ .../workflows/test-packages-action-linux.yml | 1 + 14 files changed, 107 insertions(+), 6 deletions(-) diff --git a/.github/actions/build-onedir-deps/action.yml b/.github/actions/build-onedir-deps/action.yml index c34133bc369..1d36086ec0d 100644 --- a/.github/actions/build-onedir-deps/action.yml +++ b/.github/actions/build-onedir-deps/action.yml @@ -25,7 +25,6 @@ runs: using: composite steps: - - name: Cache Deps Onedir Package Directory id: onedir-pkg-cache uses: ./.github/actions/cache diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml index ae416246d79..a11df9f02c7 100644 --- a/.github/actions/cache/action.yml +++ b/.github/actions/cache/action.yml @@ -34,7 +34,7 @@ inputs: outputs: cache-hit: description: 'A boolean value to indicate an exact match was found for the primary key' - value: ${{ steps.github-cache.outputs.cache-hit }} + value: ${{ steps.github-cache.outputs.cache-hit || steps.s3-cache.outputs.cache-hit }} runs: using: composite @@ -55,6 +55,7 @@ runs: - name: Cache Provided Path (GitHub Actions) id: github-cache + if: ${{ env.USE_S3_CACHE != 'true' }} uses: actions/cache@v4 with: path: ${{ env.GHA_CACHE_PATH }} @@ -66,11 +67,34 @@ runs: restore-keys: ${{ env.GHA_CACHE_RESTORE_KEYS }} upload-chunk-size: ${{ env.GHA_CACHE_UPLOAD_CHUNK_SIZE }} + - name: Configure AWS Credentials to access cache bucket + if: ${{ env.USE_S3_CACHE == 'true' }} + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + + - name: Cache Provided Path (S3) + if: ${{ env.USE_S3_CACHE == 'true' }} + id: s3-cache + env: + RUNS_ON_S3_BUCKET_CACHE: "salt-project-test-salt-github-actions-s3-cache" + AWS_REGION: 'us-west-2' + uses: runs-on/cache@v4 + with: + path: ${{ env.GHA_CACHE_PATH }} + key: ${{ env.GHA_CACHE_KEY }} + enableCrossOsArchive: ${{ env.GHA_CACHE_ENABLE_CROSS_OS_ARCHIVE }} + fail-on-cache-miss: ${{ env.GHA_CACHE_FAIL_ON_CACHE_MISS }} + lookup-only: ${{ env.GHA_CACHE_LOOKUP_ONLY }} + save-always: ${{ env.GHA_CACHE_SAVE_ALWAYS }} + restore-keys: ${{ env.GHA_CACHE_RESTORE_KEYS }} + upload-chunk-size: ${{ env.GHA_CACHE_UPLOAD_CHUNK_SIZE }} + - name: Verify 'fail-on-cache-miss' if: ${{ inputs.fail-on-cache-miss == 'true' }} shell: bash run: | - CACHE_HIT="${{ steps.github-cache.outputs.cache-hit }}" + CACHE_HIT="${{ steps.github-cache.outputs.cache-hit || steps.s3-cache.outputs.cache-hit }}" if [ "$CACHE_HIT" != "true" ]; then echo "No cache hit and fail-on-cache-miss is set to true." exit 1 diff --git a/.github/workflows/build-deps-ci-action.yml b/.github/workflows/build-deps-ci-action.yml index 26ad6fa8b65..b16ec5e6ee8 100644 --- a/.github/workflows/build-deps-ci-action.yml +++ b/.github/workflows/build-deps-ci-action.yml @@ -53,6 +53,8 @@ jobs: - self-hosted - linux - bastion + env: + USE_S3_CACHE: 'true' timeout-minutes: 90 strategy: fail-fast: false @@ -253,6 +255,8 @@ jobs: - self-hosted - linux - bastion + env: + USE_S3_CACHE: 'true' timeout-minutes: 90 strategy: fail-fast: false diff --git a/.github/workflows/build-deps-onedir.yml b/.github/workflows/build-deps-onedir.yml index 41d6f0260fe..8a13c7e3777 100644 --- a/.github/workflows/build-deps-onedir.yml +++ b/.github/workflows/build-deps-onedir.yml @@ -51,6 +51,8 @@ jobs: - self-hosted - linux - ${{ matrix.arch }} + env: + USE_S3_CACHE: 'true' steps: - name: "Throttle Builds" @@ -95,7 +97,8 @@ jobs: - arm64 runs-on: - ${{ matrix.arch == 'arm64' && 'macos-13-xlarge' || 'macos-12' }} - + env: + USE_S3_CACHE: 'false' steps: - name: "Throttle Builds" @@ -144,6 +147,8 @@ jobs: - x86 - amd64 runs-on: windows-latest + env: + USE_S3_CACHE: 'false' steps: - name: "Throttle Builds" diff --git a/.github/workflows/build-salt-onedir.yml b/.github/workflows/build-salt-onedir.yml index 7e3a6dc2807..7913860cf7d 100644 --- a/.github/workflows/build-salt-onedir.yml +++ b/.github/workflows/build-salt-onedir.yml @@ -41,6 +41,8 @@ jobs: build-salt-linux: name: Linux if: ${{ inputs.self-hosted-runners }} + env: + USE_S3_CACHE: 'true' strategy: fail-fast: false matrix: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6b721033fd7..d973dc456e9 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2036,6 +2036,8 @@ jobs: - self-hosted - linux - repo-nightly + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - build-source-tarball @@ -2137,6 +2139,8 @@ jobs: - self-hosted - linux - repo-nightly + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - build-pkgs-onedir @@ -2274,6 +2278,8 @@ jobs: - self-hosted - linux - repo-nightly + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - build-pkgs-onedir @@ -2462,6 +2468,8 @@ jobs: - self-hosted - linux - repo-nightly + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - build-pkgs-onedir @@ -2564,6 +2572,8 @@ jobs: - self-hosted - linux - repo-nightly + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - build-pkgs-onedir @@ -2654,6 +2664,8 @@ jobs: - self-hosted - linux - repo-nightly + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - build-salt-onedir diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9bf81b4f897..29d3bb6b49f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,6 +52,8 @@ jobs: - self-hosted - linux - repo-release + env: + USE_S3_CACHE: 'true' environment: release needs: - check-requirements @@ -125,6 +127,8 @@ jobs: - self-hosted - linux - repo-release + env: + USE_S3_CACHE: 'true' environment: release needs: - prepare-workflow @@ -191,6 +195,8 @@ jobs: - repo-release needs: - prepare-workflow + env: + USE_S3_CACHE: 'true' environment: release outputs: backup-complete: ${{ steps.backup.outputs.backup-complete }} @@ -220,12 +226,13 @@ jobs: - self-hosted - linux - repo-release + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - backup - download-onedir-artifact environment: release - steps: - name: Clone The Salt Repository uses: actions/checkout@v4 @@ -275,6 +282,8 @@ jobs: - self-hosted - linux - repo-release + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - backup @@ -389,6 +398,8 @@ jobs: - self-hosted - linux - repo-release + env: + USE_S3_CACHE: 'true' steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 159e9ca669b..5110775403b 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -1873,6 +1873,8 @@ jobs: - self-hosted - linux - repo-staging + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - build-source-tarball @@ -1974,6 +1976,8 @@ jobs: - self-hosted - linux - repo-staging + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - build-pkgs-onedir @@ -2111,6 +2115,8 @@ jobs: - self-hosted - linux - repo-staging + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - build-pkgs-onedir @@ -2301,6 +2307,8 @@ jobs: - self-hosted - linux - repo-staging + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - build-pkgs-onedir @@ -2403,6 +2411,8 @@ jobs: - self-hosted - linux - repo-staging + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - build-pkgs-onedir @@ -2493,6 +2503,8 @@ jobs: - self-hosted - linux - repo-staging + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - build-salt-onedir diff --git a/.github/workflows/templates/build-deps-ci-action.yml.jinja b/.github/workflows/templates/build-deps-ci-action.yml.jinja index 1038d2b71dd..de6dc03ba3c 100644 --- a/.github/workflows/templates/build-deps-ci-action.yml.jinja +++ b/.github/workflows/templates/build-deps-ci-action.yml.jinja @@ -53,6 +53,8 @@ jobs: - self-hosted - linux - bastion + env: + USE_S3_CACHE: 'true' timeout-minutes: 90 strategy: fail-fast: false @@ -253,6 +255,8 @@ jobs: - self-hosted - linux - bastion + env: + USE_S3_CACHE: 'true' timeout-minutes: 90 strategy: fail-fast: false diff --git a/.github/workflows/templates/build-repos.yml.jinja b/.github/workflows/templates/build-repos.yml.jinja index f1d58b38121..92f621c857e 100644 --- a/.github/workflows/templates/build-repos.yml.jinja +++ b/.github/workflows/templates/build-repos.yml.jinja @@ -17,6 +17,8 @@ - self-hosted - linux - repo-<{ gh_environment }> + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow <%- if type not in ("src", "onedir") %> diff --git a/.github/workflows/templates/release.yml.jinja b/.github/workflows/templates/release.yml.jinja index 3fedd45cad8..efeef41f8a4 100644 --- a/.github/workflows/templates/release.yml.jinja +++ b/.github/workflows/templates/release.yml.jinja @@ -74,6 +74,8 @@ permissions: - self-hosted - linux - repo-<{ gh_environment }> + env: + USE_S3_CACHE: 'true' environment: <{ gh_environment }> <%- if prepare_workflow_needs %> needs: @@ -157,6 +159,8 @@ permissions: - self-hosted - linux - repo-<{ gh_environment }> + env: + USE_S3_CACHE: 'true' environment: <{ gh_environment }> needs: - prepare-workflow @@ -212,6 +216,8 @@ permissions: - repo-<{ gh_environment }> needs: - prepare-workflow + env: + USE_S3_CACHE: 'true' environment: <{ gh_environment }> outputs: backup-complete: ${{ steps.backup.outputs.backup-complete }} @@ -242,12 +248,13 @@ permissions: - self-hosted - linux - repo-<{ gh_environment }> + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - backup - download-onedir-artifact environment: <{ gh_environment }> - steps: - name: Clone The Salt Repository uses: actions/checkout@v4 @@ -282,6 +289,8 @@ permissions: - self-hosted - linux - repo-<{ gh_environment }> + env: + USE_S3_CACHE: 'true' needs: - prepare-workflow - backup @@ -396,6 +405,8 @@ permissions: - self-hosted - linux - repo-<{ gh_environment }> + env: + USE_S3_CACHE: 'true' needs: - backup - release @@ -437,6 +448,8 @@ permissions: - self-hosted - linux - repo-<{ gh_environment }> + env: + USE_S3_CACHE: 'true' steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/templates/test-package-downloads-action.yml.jinja b/.github/workflows/templates/test-package-downloads-action.yml.jinja index 979dba27512..ee5f5fbfc1b 100644 --- a/.github/workflows/templates/test-package-downloads-action.yml.jinja +++ b/.github/workflows/templates/test-package-downloads-action.yml.jinja @@ -61,6 +61,8 @@ jobs: - self-hosted - linux - bastion + env: + USE_S3_CACHE: 'true' environment: ${{ inputs.environment }} timeout-minutes: 120 # 2 Hours - More than this and something is wrong strategy: @@ -270,6 +272,8 @@ jobs: macos: name: MacOS runs-on: ${{ matrix.distro-slug }} + env: + USE_S3_CACHE: 'false' environment: ${{ inputs.environment }} timeout-minutes: 120 # 2 Hours - More than this and something is wrong strategy: @@ -469,6 +473,8 @@ jobs: windows: name: Windows + env: + USE_S3_CACHE: 'true' runs-on: - self-hosted - linux diff --git a/.github/workflows/test-package-downloads-action.yml b/.github/workflows/test-package-downloads-action.yml index 93d43fc4b76..dc3fb12a661 100644 --- a/.github/workflows/test-package-downloads-action.yml +++ b/.github/workflows/test-package-downloads-action.yml @@ -61,6 +61,8 @@ jobs: - self-hosted - linux - bastion + env: + USE_S3_CACHE: 'true' environment: ${{ inputs.environment }} timeout-minutes: 120 # 2 Hours - More than this and something is wrong strategy: @@ -379,6 +381,8 @@ jobs: macos: name: MacOS runs-on: ${{ matrix.distro-slug }} + env: + USE_S3_CACHE: 'false' environment: ${{ inputs.environment }} timeout-minutes: 120 # 2 Hours - More than this and something is wrong strategy: @@ -585,6 +589,8 @@ jobs: windows: name: Windows + env: + USE_S3_CACHE: 'true' runs-on: - self-hosted - linux diff --git a/.github/workflows/test-packages-action-linux.yml b/.github/workflows/test-packages-action-linux.yml index ae85b3989ac..a2dc20f0080 100644 --- a/.github/workflows/test-packages-action-linux.yml +++ b/.github/workflows/test-packages-action-linux.yml @@ -69,6 +69,7 @@ env: PIP_EXTRA_INDEX_URL: https://pypi.org/simple PIP_DISABLE_PIP_VERSION_CHECK: "1" RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1" + USE_S3_CACHE: 'true' jobs: From 64bc4560a905a0863079e9f89609d424402b3e38 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 18 Mar 2024 18:44:18 +0000 Subject: [PATCH 5/5] Get AWS regions and SPB_ENVIRONMENT from metadata endpoint --- .github/actions/cache/action.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml index a11df9f02c7..020b9d1e6b8 100644 --- a/.github/actions/cache/action.yml +++ b/.github/actions/cache/action.yml @@ -67,18 +67,29 @@ runs: restore-keys: ${{ env.GHA_CACHE_RESTORE_KEYS }} upload-chunk-size: ${{ env.GHA_CACHE_UPLOAD_CHUNK_SIZE }} + - name: Get Salt Project GitHub Actions Bot Environment + if: ${{ env.USE_S3_CACHE == 'true' }} + shell: bash + run: | + TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") + SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) + echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" | tee -a "$GITHUB_ENV" + REGION=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/region) + echo "GHA_CACHE_AWS_REGION=$REGION" | tee -a "$GITHUB_ENV" + - name: Configure AWS Credentials to access cache bucket + id: creds if: ${{ env.USE_S3_CACHE == 'true' }} uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: us-west-2 + aws-region: ${{ env.GHA_CACHE_AWS_REGION }} - name: Cache Provided Path (S3) if: ${{ env.USE_S3_CACHE == 'true' }} id: s3-cache env: - RUNS_ON_S3_BUCKET_CACHE: "salt-project-test-salt-github-actions-s3-cache" - AWS_REGION: 'us-west-2' + AWS_REGION: ${{ env.GHA_CACHE_AWS_REGION }} + RUNS_ON_S3_BUCKET_CACHE: salt-project-${{ env.SPB_ENVIRONMENT}}-salt-github-actions-s3-cache uses: runs-on/cache@v4 with: path: ${{ env.GHA_CACHE_PATH }}