Initial commit of s3 caching

.github/actions/build-onedir-deps/action.yml

@@ -25,7 +25,6 @@ runs:
   using: composite
 
   steps:
-
     - name: Cache Deps Onedir Package Directory
       id: onedir-pkg-cache
       uses: ./.github/actions/cache

.github/actions/cache/action.yml

@@ -34,7 +34,7 @@ inputs:
 outputs:
   cache-hit:
     description: 'A boolean value to indicate an exact match was found for the primary key'
-    value: ${{ steps.github-cache.outputs.cache-hit }}
+    value: ${{ steps.github-cache.outputs.cache-hit || steps.s3-cache.outputs.cache-hit }}
 
 runs:
   using: composite
@@ -55,6 +55,7 @@ runs:
 
     - name: Cache Provided Path (GitHub Actions)
       id: github-cache
+      if: ${{ env.USE_S3_CACHE != 'true' }}
       uses: actions/cache@v4
       with:
         path: ${{ env.GHA_CACHE_PATH }}
@@ -66,11 +67,34 @@ runs:
         restore-keys: ${{ env.GHA_CACHE_RESTORE_KEYS }}
         upload-chunk-size: ${{ env.GHA_CACHE_UPLOAD_CHUNK_SIZE }}
 
+    - name: Configure AWS Credentials to access cache bucket
+      if: ${{ env.USE_S3_CACHE == 'true' }}
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-region: us-west-2
+
+    - name: Cache Provided Path (S3)
+      if: ${{ env.USE_S3_CACHE == 'true' }}
+      id: s3-cache
+      env:
+        RUNS_ON_S3_BUCKET_CACHE: "salt-project-test-salt-github-actions-s3-cache"
+        AWS_REGION: 'us-west-2'
+      uses: runs-on/cache@v4
+      with:
+        path: ${{ env.GHA_CACHE_PATH }}
+        key: ${{ env.GHA_CACHE_KEY }}
+        enableCrossOsArchive: ${{ env.GHA_CACHE_ENABLE_CROSS_OS_ARCHIVE }}
+        fail-on-cache-miss: ${{ env.GHA_CACHE_FAIL_ON_CACHE_MISS }}
+        lookup-only: ${{ env.GHA_CACHE_LOOKUP_ONLY }}
+        save-always: ${{ env.GHA_CACHE_SAVE_ALWAYS }}
+        restore-keys: ${{ env.GHA_CACHE_RESTORE_KEYS }}
+        upload-chunk-size: ${{ env.GHA_CACHE_UPLOAD_CHUNK_SIZE }}
+
     - name: Verify 'fail-on-cache-miss'
       if: ${{ inputs.fail-on-cache-miss == 'true' }}
       shell: bash
       run: |
-        CACHE_HIT="${{ steps.github-cache.outputs.cache-hit }}"
+        CACHE_HIT="${{ steps.github-cache.outputs.cache-hit || steps.s3-cache.outputs.cache-hit }}"
         if [ "$CACHE_HIT" != "true" ]; then
           echo "No cache hit and fail-on-cache-miss is set to true."
           exit 1

.github/workflows/build-deps-ci-action.yml

@@ -53,6 +53,8 @@ jobs:
       - self-hosted
       - linux
       - bastion
+    env:
+      USE_S3_CACHE: 'true'
     timeout-minutes: 90
     strategy:
       fail-fast: false
@@ -253,6 +255,8 @@ jobs:
       - self-hosted
       - linux
       - bastion
+    env:
+      USE_S3_CACHE: 'true'
     timeout-minutes: 90
     strategy:
       fail-fast: false

.github/workflows/build-deps-onedir.yml

@@ -51,6 +51,8 @@ jobs:
       - self-hosted
       - linux
       - ${{ matrix.arch }}
+    env:
+      USE_S3_CACHE: 'true'
     steps:
 
       - name: "Throttle Builds"
@@ -95,7 +97,8 @@ jobs:
           - arm64
     runs-on:
       - ${{ matrix.arch == 'arm64' && 'macos-13-xlarge' || 'macos-12' }}
-
+    env:
+      USE_S3_CACHE: 'false'
     steps:
 
       - name: "Throttle Builds"
@@ -144,6 +147,8 @@ jobs:
           - x86
           - amd64
     runs-on: windows-latest
+    env:
+      USE_S3_CACHE: 'false'
     steps:
 
       - name: "Throttle Builds"

.github/workflows/build-salt-onedir.yml

@@ -41,6 +41,8 @@ jobs:
   build-salt-linux:
     name: Linux
     if: ${{ inputs.self-hosted-runners }}
+    env:
+      USE_S3_CACHE: 'true'
     strategy:
       fail-fast: false
       matrix:

.github/workflows/nightly.yml

@@ -2007,6 +2007,8 @@ jobs:
       - self-hosted
       - linux
       - repo-nightly
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - build-source-tarball
@@ -2107,6 +2109,8 @@ jobs:
       - self-hosted
       - linux
       - repo-nightly
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - build-pkgs-onedir
@@ -2244,6 +2248,8 @@ jobs:
       - self-hosted
       - linux
       - repo-nightly
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - build-pkgs-onedir
@@ -2432,6 +2438,8 @@ jobs:
       - self-hosted
       - linux
       - repo-nightly
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - build-pkgs-onedir
@@ -2534,6 +2542,8 @@ jobs:
       - self-hosted
       - linux
       - repo-nightly
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - build-pkgs-onedir
@@ -2624,6 +2634,8 @@ jobs:
       - self-hosted
       - linux
       - repo-nightly
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - build-salt-onedir

.github/workflows/release.yml

@@ -52,6 +52,8 @@ jobs:
       - self-hosted
       - linux
       - repo-release
+    env:
+      USE_S3_CACHE: 'true'
     environment: release
     needs:
       - check-requirements
@@ -125,6 +127,8 @@ jobs:
       - self-hosted
       - linux
       - repo-release
+    env:
+      USE_S3_CACHE: 'true'
     environment: release
     needs:
       - prepare-workflow
@@ -191,6 +195,8 @@ jobs:
       - repo-release
     needs:
       - prepare-workflow
+    env:
+      USE_S3_CACHE: 'true'
     environment: release
     outputs:
       backup-complete: ${{ steps.backup.outputs.backup-complete }}
@@ -220,12 +226,13 @@ jobs:
       - self-hosted
       - linux
       - repo-release
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - backup
       - download-onedir-artifact
     environment: release
-
     steps:
       - name: Clone The Salt Repository
         uses: actions/checkout@v4
@@ -275,6 +282,8 @@ jobs:
       - self-hosted
       - linux
       - repo-release
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - backup
@@ -389,6 +398,8 @@ jobs:
       - self-hosted
       - linux
       - repo-release
+    env:
+      USE_S3_CACHE: 'true'
     steps:
       - uses: actions/checkout@v4
 

.github/workflows/staging.yml

@@ -1852,6 +1852,8 @@ jobs:
       - self-hosted
       - linux
       - repo-staging
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - build-source-tarball
@@ -1952,6 +1954,8 @@ jobs:
       - self-hosted
       - linux
       - repo-staging
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - build-pkgs-onedir
@@ -2089,6 +2093,8 @@ jobs:
       - self-hosted
       - linux
       - repo-staging
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - build-pkgs-onedir
@@ -2279,6 +2285,8 @@ jobs:
       - self-hosted
       - linux
       - repo-staging
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - build-pkgs-onedir
@@ -2381,6 +2389,8 @@ jobs:
       - self-hosted
       - linux
       - repo-staging
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - build-pkgs-onedir
@@ -2471,6 +2481,8 @@ jobs:
       - self-hosted
       - linux
       - repo-staging
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - build-salt-onedir

.github/workflows/templates/build-deps-ci-action.yml.jinja

@@ -53,6 +53,8 @@ jobs:
       - self-hosted
       - linux
       - bastion
+    env:
+      USE_S3_CACHE: 'true'
     timeout-minutes: 90
     strategy:
       fail-fast: false
@@ -253,6 +255,8 @@ jobs:
       - self-hosted
       - linux
       - bastion
+    env:
+      USE_S3_CACHE: 'true'
     timeout-minutes: 90
     strategy:
       fail-fast: false

.github/workflows/templates/build-repos.yml.jinja

@@ -17,6 +17,8 @@
       - self-hosted
       - linux
       - repo-<{ gh_environment }>
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       <%- if type not in ("src", "onedir") %>

.github/workflows/templates/release.yml.jinja

@@ -74,6 +74,8 @@ permissions:
       - self-hosted
       - linux
       - repo-<{ gh_environment }>
+    env:
+      USE_S3_CACHE: 'true'
     environment: <{ gh_environment }>
     <%- if prepare_workflow_needs %>
     needs:
@@ -157,6 +159,8 @@ permissions:
       - self-hosted
       - linux
       - repo-<{ gh_environment }>
+    env:
+      USE_S3_CACHE: 'true'
     environment: <{ gh_environment }>
     needs:
       - prepare-workflow
@@ -212,6 +216,8 @@ permissions:
       - repo-<{ gh_environment }>
     needs:
       - prepare-workflow
+    env:
+      USE_S3_CACHE: 'true'
     environment: <{ gh_environment }>
     outputs:
       backup-complete: ${{ steps.backup.outputs.backup-complete }}
@@ -242,12 +248,13 @@ permissions:
       - self-hosted
       - linux
       - repo-<{ gh_environment }>
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - backup
       - download-onedir-artifact
     environment: <{ gh_environment }>
-
     steps:
       - name: Clone The Salt Repository
         uses: actions/checkout@v4
@@ -282,6 +289,8 @@ permissions:
       - self-hosted
       - linux
       - repo-<{ gh_environment }>
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - prepare-workflow
       - backup
@@ -396,6 +405,8 @@ permissions:
       - self-hosted
       - linux
       - repo-<{ gh_environment }>
+    env:
+      USE_S3_CACHE: 'true'
     needs:
       - backup
       - release
@@ -437,6 +448,8 @@ permissions:
       - self-hosted
       - linux
       - repo-<{ gh_environment }>
+    env:
+      USE_S3_CACHE: 'true'
     steps:
       - uses: actions/checkout@v4
 

.github/workflows/templates/test-package-downloads-action.yml.jinja

@@ -61,6 +61,8 @@ jobs:
       - self-hosted
       - linux
       - bastion
+    env:
+      USE_S3_CACHE: 'true'
     environment: ${{ inputs.environment }}
     timeout-minutes: 120  # 2 Hours - More than this and something is wrong
     strategy:
@@ -270,6 +272,8 @@ jobs:
   macos:
     name: MacOS
     runs-on: ${{ matrix.distro-slug }}
+    env:
+      USE_S3_CACHE: 'false'
     environment: ${{ inputs.environment }}
     timeout-minutes: 120  # 2 Hours - More than this and something is wrong
     strategy:
@@ -469,6 +473,8 @@ jobs:
 
   windows:
     name: Windows
+    env:
+      USE_S3_CACHE: 'true'
     runs-on:
       - self-hosted
       - linux

.github/workflows/test-package-downloads-action.yml

@@ -61,6 +61,8 @@ jobs:
       - self-hosted
       - linux
       - bastion
+    env:
+      USE_S3_CACHE: 'true'
     environment: ${{ inputs.environment }}
     timeout-minutes: 120  # 2 Hours - More than this and something is wrong
     strategy:
@@ -379,6 +381,8 @@ jobs:
   macos:
     name: MacOS
     runs-on: ${{ matrix.distro-slug }}
+    env:
+      USE_S3_CACHE: 'false'
     environment: ${{ inputs.environment }}
     timeout-minutes: 120  # 2 Hours - More than this and something is wrong
     strategy:
@@ -585,6 +589,8 @@ jobs:
 
   windows:
     name: Windows
+    env:
+      USE_S3_CACHE: 'true'
     runs-on:
       - self-hosted
       - linux

.github/workflows/test-packages-action-linux.yml

@@ -69,6 +69,7 @@ env:
   PIP_EXTRA_INDEX_URL: https://pypi.org/simple
   PIP_DISABLE_PIP_VERSION_CHECK: "1"
   RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
+  USE_S3_CACHE: 'true'
 
 jobs: