From 3fb1279f321d05447b5da172df374e80b29ed470 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 13 Jan 2024 20:29:52 +0000 Subject: [PATCH] Allow excluding paths when cleaning up archives Signed-off-by: Pedro Algarvio --- .github/workflows/ci.yml | 2 +- .github/workflows/nightly.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/scheduled.yml | 2 +- .github/workflows/staging.yml | 2 +- .github/workflows/templates/layout.yml.jinja | 3 ++- .github/workflows/templates/release.yml.jinja | 2 +- pkg/common/env-cleanup-rules.yml | 15 ++++++++++++ tools/pkg/__init__.py | 24 +++++++++++++++++++ 9 files changed, 47 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9cfe4c5780..65cacd83e73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -169,7 +169,7 @@ jobs: - name: Get Hash For Nox Tarball Cache id: nox-archive-hash run: | - echo "nox-archive-hash=${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}" | tee -a "$GITHUB_OUTPUT" + echo "nox-archive-hash=${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py', 'pkg/common/env-cleanup-rules.yml') }}" | tee -a "$GITHUB_OUTPUT" - name: Write Changed Files To A Local File run: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1c56893b4c3..c4716979cb7 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -221,7 +221,7 @@ jobs: - name: Get Hash For Nox Tarball Cache id: nox-archive-hash run: | - echo "nox-archive-hash=${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}" | tee -a "$GITHUB_OUTPUT" + echo "nox-archive-hash=${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py', 'pkg/common/env-cleanup-rules.yml') }}" | tee -a "$GITHUB_OUTPUT" - name: Write Changed Files To A Local File run: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 38cf3133d17..e4bd8a57550 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -115,7 +115,7 @@ jobs: - name: Get Hash For Nox Tarball Cache id: nox-archive-hash run: | - echo "nox-archive-hash=${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}" | tee -a "$GITHUB_OUTPUT" + echo "nox-archive-hash=${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py', 'pkg/common/env-cleanup-rules.yml') }}" | tee -a "$GITHUB_OUTPUT" download-onedir-artifact: name: Download Staging Onedir Artifact diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 4a6cf541814..c5e0d89a61a 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -211,7 +211,7 @@ jobs: - name: Get Hash For Nox Tarball Cache id: nox-archive-hash run: | - echo "nox-archive-hash=${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}" | tee -a "$GITHUB_OUTPUT" + echo "nox-archive-hash=${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py', 'pkg/common/env-cleanup-rules.yml') }}" | tee -a "$GITHUB_OUTPUT" - name: Write Changed Files To A Local File run: diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 9b52e9c6d7d..b96199f880f 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -199,7 +199,7 @@ jobs: - name: Get Hash For Nox Tarball Cache id: nox-archive-hash run: | - echo "nox-archive-hash=${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}" | tee -a "$GITHUB_OUTPUT" + echo "nox-archive-hash=${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py', 'pkg/common/env-cleanup-rules.yml') }}" | tee -a "$GITHUB_OUTPUT" - name: Check Existing Releases env: diff --git a/.github/workflows/templates/layout.yml.jinja b/.github/workflows/templates/layout.yml.jinja index 5d42133ce77..6316a8aca7d 100644 --- a/.github/workflows/templates/layout.yml.jinja +++ b/.github/workflows/templates/layout.yml.jinja @@ -9,6 +9,7 @@ <%- set gpg_key_id = "64CBBC8173D76B3F" %> <%- set prepare_actual_release = prepare_actual_release | default(False) %> <%- set gh_actions_workflows_python_version = "3.10" %> +<%- set nox_archive_hashfiles = "${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py', 'pkg/common/env-cleanup-rules.yml') }}" %> --- <%- block name %> name: <{ workflow_name }> @@ -216,7 +217,7 @@ jobs: - name: Get Hash For Nox Tarball Cache id: nox-archive-hash run: | - echo "nox-archive-hash=${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}" | tee -a "$GITHUB_OUTPUT" + echo "nox-archive-hash=<{ nox_archive_hashfiles }>" | tee -a "$GITHUB_OUTPUT" <%- if prepare_actual_release %> diff --git a/.github/workflows/templates/release.yml.jinja b/.github/workflows/templates/release.yml.jinja index 704e2ec940b..7be119e57c8 100644 --- a/.github/workflows/templates/release.yml.jinja +++ b/.github/workflows/templates/release.yml.jinja @@ -143,7 +143,7 @@ permissions: - name: Get Hash For Nox Tarball Cache id: nox-archive-hash run: | - echo "nox-archive-hash=${{ hashFiles('requirements/**/*.txt', 'cicd/golden-images.json', 'noxfile.py') }}" | tee -a "$GITHUB_OUTPUT" + echo "nox-archive-hash=<{ nox_archive_hashfiles }>" | tee -a "$GITHUB_OUTPUT" <%- endblock prepare_workflow_job %> <%- endif %> diff --git a/pkg/common/env-cleanup-rules.yml b/pkg/common/env-cleanup-rules.yml index 09708110473..43b4a628af2 100644 --- a/pkg/common/env-cleanup-rules.yml +++ b/pkg/common/env-cleanup-rules.yml @@ -1,5 +1,8 @@ --- common: + exclude_patterns: &common_exclude_patterns + - "**/site-packages/ansible/plugins/test" + - "**/site-packages/ansible/plugins/test/**" dir_patterns: &common_dir_patterns - "**/__pycache__" - "**/lib/python3.*/test" @@ -24,16 +27,22 @@ common: ci: darwin: + exclude_patterns: &ci_darwin_exclude_patterns + - *common_exclude_patterns dir_patterns: &ci_darwin_dir_patterns - *common_dir_patterns file_patterns: &ci_darwin_file_patterns - *common_file_patterns linux: + exclude_patterns: &ci_linux_exclude_patterns + - *common_exclude_patterns dir_patterns: &ci_linux_dir_patterns - *common_dir_patterns file_patterns: &ci_linux_file_patterns - *common_file_patterns windows: + exclude_patterns: &ci_windows_exclude_patterns + - *common_exclude_patterns dir_patterns: &ci_windows_dir_patterns - *common_dir_patterns - "**/artifacts/salt/configs" @@ -52,6 +61,8 @@ ci: pkg: darwin: + exclude_patterns: + - *ci_darwin_exclude_patterns dir_patterns: - *ci_darwin_dir_patterns - "**/pkgconfig" @@ -62,11 +73,15 @@ pkg: file_patterns: - *ci_darwin_file_patterns linux: + exclude_patterns: + - *ci_linux_exclude_patterns dir_patterns: - *ci_linux_dir_patterns file_patterns: - *ci_linux_file_patterns windows: + exclude_patterns: + - *ci_windows_exclude_patterns dir_patterns: - *ci_windows_dir_patterns - "**/salt/share" diff --git a/tools/pkg/__init__.py b/tools/pkg/__init__.py index 05612996655..2d843893640 100644 --- a/tools/pkg/__init__.py +++ b/tools/pkg/__init__.py @@ -257,6 +257,10 @@ def pre_archive_cleanup(ctx: Context, cleanup_path: str, pkg: bool = False): else: yield patterns + exclude_patterns = set() + for pattern in unnest_lists(patterns["exclude_patterns"]): + exclude_patterns.add(pattern) + dir_patterns = set() for pattern in unnest_lists(patterns["dir_patterns"]): dir_patterns.add(pattern) @@ -271,6 +275,16 @@ def pre_archive_cleanup(ctx: Context, cleanup_path: str, pkg: bool = False): if not path.exists(): continue match_path = path.as_posix() + skip_match = False + for pattern in exclude_patterns: + if fnmatch.fnmatch(str(match_path), pattern): + ctx.info( + f"Excluded file: {match_path}; Matching pattern: {pattern!r}" + ) + skip_match = True + break + if skip_match: + continue for pattern in dir_patterns: if fnmatch.fnmatch(str(match_path), pattern): ctx.info( @@ -283,6 +297,16 @@ def pre_archive_cleanup(ctx: Context, cleanup_path: str, pkg: bool = False): if not path.exists(): continue match_path = path.as_posix() + skip_match = False + for pattern in exclude_patterns: + if fnmatch.fnmatch(str(match_path), pattern): + ctx.info( + f"Excluded file: {match_path}; Matching pattern: {pattern!r}" + ) + skip_match = True + break + if skip_match: + continue for pattern in file_patterns: if fnmatch.fnmatch(str(match_path), pattern): ctx.info(