saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-16 17:50:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Clearly separate each step on prepare-ci

Browse source 
Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>
This commit is contained in:
Pedro Algarvio 2023-01-18 10:34:52 +00:00 • committed by Pedro Algarvio
parent a26e12b412
commit 1f72058d96
3 changed files with 121 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
.github/workflows/ci.yml vendored
View file

@ -45,7 +45,7 @@ jobs:
name: Prepare CI
runs-on: ubuntu-latest
outputs:
jobs: ${{ steps.process-changed-files.outputs.jobs }}
jobs: ${{ steps.define-jobs.outputs.jobs }}
changed-files: ${{ steps.process-changed-files.outputs.changed-files }}
testrun: ${{ steps.define-testrun.outputs.testrun }}
salt-version: ${{ steps.setup-salt-version.outputs.salt-version }}
@ -54,7 +54,9 @@ jobs:
- uses: actions/checkout@v3
with:
fetch-depth: 0 # Full clone to also get the tags to get the right salt version
- name: Get Changed Files
if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}}
id: changed-files
uses: dorny/paths-filter@v2
with:
@ -123,6 +125,10 @@ jobs:
- name: Setup Python Tools Scripts
uses: ./.github/actions/setup-python-tools-scripts
- name: Pretty Print The GH Actions Event
run:
tools ci print-gh-event
- name: Setup Salt Version
id: setup-salt-version
uses: ./.github/actions/setup-salt-version
@ -130,27 +136,35 @@ jobs:
salt-version: "${{ inputs.salt-version }}"
- name: Write Changed Files To A Local File
if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}}
run:
echo '${{ toJSON(steps.changed-files.outputs) }}' > changed-files.json
- name: Check Local Changed Files Contents
if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}}
run:
cat changed-files.json
- name: Process Changed Files
if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}}
id: process-changed-files
run:
tools ci process-changed-files ${{ github.event_name }} changed-files.json
- name: Check Collected Jobs
run:
echo '${{ steps.process-changed-files.outputs.jobs }}' | jq -C '.'
- name: Check Collected Changed Files
if: ${{ github.event_name != 'schedule' }}
if: ${{ github.event_name != 'schedule' && github.event_name != 'push'}}
run:
echo '${{ steps.process-changed-files.outputs.changed-files }}' | jq -C '.'
- name: Define Jobs To Run
id: define-jobs
run:
tools ci define-jobs ${{ github.event_name }}
- name: Check Collected Jobs
run:
echo '${{ steps.define-jobs.outputs.jobs }}' | jq -C '.'
- name: Define Testrun
id: define-testrun
run:

2
.github/workflows/pre-commit-action.yml vendored
View file

@ -52,7 +52,7 @@ jobs:
pre-commit run --show-diff-on-failure --color=always --files ${{ join(fromJSON(inputs.changed-files)['repo_files'], ' ') }}
- name: Check Docs On Deleted Files
if: fromJSON(inputs.changed-files)['deleted']
if: github.event_name == 'pull_request' && fromJSON(inputs.changed-files)['deleted']
env:
PIP_EXTRA_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
run: |

170
tools/ci.py
View file

@ -21,6 +21,32 @@ REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent
ci = command_group(name="ci", help="CI Related Commands", description=__doc__)
@ci.command(
name="print-gh-event",
)
def print_gh_event(ctx: Context):
"""
Pretty print the GH Actions event.
"""
gh_event_path = os.environ.get("GITHUB_EVENT_PATH") or None
if gh_event_path is None:
ctx.warn("The 'GITHUB_EVENT_PATH' variable is not set.")
ctx.exit(1)
if TYPE_CHECKING:
assert gh_event_path is not None
try:
gh_event = json.loads(open(gh_event_path).read())
except Exception as exc:
ctx.error(f"Could not load the GH Event payload from {gh_event_path!r}:\n", exc)
ctx.exit(1)
ctx.info("GH Event Payload:")
ctx.print(gh_event, soft_wrap=True)
ctx.exit(0)
@ci.command(
name="process-changed-files",
arguments={
@ -36,6 +62,67 @@ ci = command_group(name="ci", help="CI Related Commands", description=__doc__)
},
)
def process_changed_files(ctx: Context, event_name: str, changed_files: pathlib.Path):
"""
Process changed files to avoid path traversal.
"""
github_output = os.environ.get("GITHUB_OUTPUT")
if github_output is None:
ctx.warn("The 'GITHUB_OUTPUT' variable is not set.")
ctx.exit(1)
if TYPE_CHECKING:
assert github_output is not None
if not changed_files.exists():
ctx.error(f"The '{changed_files}' file does not exist.")
ctx.exit(1)
try:
changed_files_contents = json.loads(changed_files.read_text())
except Exception as exc:
ctx.error(f"Could not load the changed files from '{changed_files}': {exc}")
ctx.exit(1)
sanitized_changed_files = {}
ctx.info("Sanitizing paths and confirming no path traversal is being used...")
for key, data in changed_files_contents.items():
try:
loaded_data = json.loads(data)
except ValueError:
loaded_data = data
if key.endswith("_files"):
files = set()
for entry in list(loaded_data):
if not entry:
loaded_data.remove(entry)
try:
entry = REPO_ROOT.joinpath(entry).resolve().relative_to(REPO_ROOT)
except ValueError:
ctx.error(
f"While processing the changed files key {key!r}, the "
f"path entry {entry!r} was checked and it's not relative "
"to the repository root."
)
ctx.exit(1)
files.add(str(entry))
sanitized_changed_files[key] = sorted(files)
continue
sanitized_changed_files[key] = loaded_data
ctx.info("Writing 'changed-files' to the github outputs file")
with open(github_output, "a", encoding="utf-8") as wfh:
wfh.write(f"changed-files={json.dumps(sanitized_changed_files)}\n")
ctx.exit(0)
@ci.command(
name="define-jobs",
arguments={
"event_name": {
"help": "The name of the GitHub event being processed.",
},
},
)
def define_jobs(ctx: Context, event_name: str):
"""
Set GH Actions outputs for what should build or not.
"""
@ -66,48 +153,6 @@ def process_changed_files(ctx: Context, event_name: str, changed_files: pathlib.
# Let's it print until the end
time.sleep(1)
if not changed_files.exists():
ctx.error(f"The '{changed_files}' file does not exist.")
ctx.exit(1)
try:
changed_files_contents = json.loads(changed_files.read_text())
except Exception as exc:
ctx.error(f"Could not load the changed files from '{changed_files}': {exc}")
ctx.exit(1)
sanitized_changed_files = {}
if event_name != "schedule":
ctx.info("Sanitizing paths and confirming no path traversal is being used...")
for key, data in changed_files_contents.items():
try:
loaded_data = json.loads(data)
except ValueError:
loaded_data = data
if key.endswith("_files"):
files = set()
for entry in list(loaded_data):
if not entry:
loaded_data.remove(entry)
try:
entry = (
REPO_ROOT.joinpath(entry).resolve().relative_to(REPO_ROOT)
)
except ValueError:
ctx.error(
f"While processing the changed files key {key!r}, the "
f"path entry {entry!r} was checked and it's not relative "
"to the repository root."
)
ctx.exit(1)
files.add(str(entry))
sanitized_changed_files[key] = sorted(files)
continue
sanitized_changed_files[key] = loaded_data
ctx.info("Writing 'changed-files' to the github outputs file")
with open(github_output, "a", encoding="utf-8") as wfh:
wfh.write(f"changed-files={json.dumps(sanitized_changed_files)}\n")
ctx.info("Selecting which type of jobs(self hosted runners or not) to run")
jobs = {"github-hosted-runners": False, "self-hosted-runners": False}
if event_name == "pull_request":
@ -171,14 +216,6 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path):
"""
Set GH Actions outputs for what and how Salt should be tested.
"""
gh_event_path = os.environ.get("GITHUB_EVENT_PATH") or None
if gh_event_path is None:
ctx.warn("The 'GITHUB_EVENT_PATH' variable is not set.")
ctx.exit(1)
if TYPE_CHECKING:
assert gh_event_path is not None
github_output = os.environ.get("GITHUB_OUTPUT")
if github_output is None:
ctx.warn("The 'GITHUB_OUTPUT' variable is not set.")
@ -187,17 +224,6 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path):
if TYPE_CHECKING:
assert github_output is not None
try:
gh_event = json.loads(open(gh_event_path).read())
except Exception as exc:
ctx.error(f"Could not load the GH Event payload from {gh_event_path!r}:\n", exc)
ctx.exit(1)
ctx.info("GH Event Payload:")
ctx.print(gh_event, soft_wrap=True)
# Let it print until the end
time.sleep(1)
github_step_summary = os.environ.get("GITHUB_STEP_SUMMARY")
if github_step_summary is None:
ctx.warn("The 'GITHUB_STEP_SUMMARY' variable is not set.")
@ -206,15 +232,6 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path):
if TYPE_CHECKING:
assert github_step_summary is not None
if not changed_files.exists():
ctx.error(f"The '{changed_files}' file does not exist.")
ctx.exit(1)
try:
changed_files_contents = json.loads(changed_files.read_text())
except Exception as exc:
ctx.error(f"Could not load the changed files from '{changed_files}': {exc}")
ctx.exit(1)
if event_name in ("push", "schedule"):
# In this case, a full test run is in order
ctx.info("Writing 'testrun' to the github outputs file")
@ -226,6 +243,19 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path):
wfh.write(f"Full test run chosen due to event type of {event_name!r}.\n")
return
if not changed_files.exists():
ctx.error(f"The '{changed_files}' file does not exist.")
ctx.error(
"FYI, the command 'tools process-changed-files <changed-files-path>' "
"needs to run prior to this one."
)
ctx.exit(1)
try:
changed_files_contents = json.loads(changed_files.read_text())
except Exception as exc:
ctx.error(f"Could not load the changed files from '{changed_files}': {exc}")
ctx.exit(1)
# So, it's a pull request...
# Based on which files changed, or other things like PR comments we can
# decide what to run, or even if the full test run should be running on the