From 4b02270161604917d1e6d43ec6d89d9d350e7308 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 29 Sep 2023 18:38:31 +0100 Subject: [PATCH 01/31] Create the XML coverage reports using the onedir Signed-off-by: Pedro Algarvio --- noxfile.py | 17 +++++++++++++++-- tools/vm.py | 4 +++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index cc3723c44f3..16d5ceb5386 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1381,8 +1381,7 @@ def create_html_coverage_report(session): ) -@nox.session(python="3", name="create-xml-coverage-reports") -def create_xml_coverage_reports(session): +def _create_xml_coverage_reports(session): _install_coverage_requirement(session) env = { # The full path to the .coverage data file. Makes sure we always write @@ -1419,6 +1418,20 @@ def create_xml_coverage_reports(session): session_warn(session, "Failed to generate the source XML code coverage report") +@nox.session(python="3", name="create-xml-coverage-reports") +def create_xml_coverage_reports(session): + _create_xml_coverage_reports(session) + + +@nox.session( + python=str(ONEDIR_PYTHON_PATH), + name="create-xml-coverage-reports-onedir", + venv_params=["--system-site-packages"], +) +def create_xml_coverage_reports_onedir(session): + _create_xml_coverage_reports(session) + + class Tee: """ Python class to mimic linux tee behaviour diff --git a/tools/vm.py b/tools/vm.py index ad45dbed6f0..000abc4e40f 100644 --- a/tools/vm.py +++ b/tools/vm.py @@ -1439,7 +1439,9 @@ class VM: """ Create XML coverage reports """ - return self.run_nox("create-xml-coverage-reports", session_args=[self.name]) + return self.run_nox( + "create-xml-coverage-reports-onedir", session_args=[self.name] + ) def compress_dependencies(self): """ From 22178c5be1149294b43cc8365e9ec38945b381dd Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 29 Sep 2023 18:46:53 +0100 Subject: [PATCH 02/31] We need XML coverage reports for codedov. Remove the step for now. Signed-off-by: Pedro Algarvio --- .github/workflows/test-action-macos.yml | 5 ----- .github/workflows/test-action.yml | 5 ----- .github/workflows/test-package-downloads-action-linux.yml | 5 ----- .github/workflows/test-package-downloads-action-macos.yml | 5 ----- .github/workflows/test-package-downloads-action-windows.yml | 5 ----- 5 files changed, 25 deletions(-) diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index dd12938561e..6e4c769953f 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -316,11 +316,6 @@ jobs: run: | nox -e combine-coverage - - name: Create XML Coverage Reports - if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled' - run: | - nox -e create-xml-coverage-reports - - name: Prepare Test Run Artifacts id: download-artifacts-from-vm if: always() && job.status != 'cancelled' diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index 0cb10a4aa8d..da5e6bbe4bc 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -257,11 +257,6 @@ jobs: run: | tools --timestamps vm combine-coverage ${{ inputs.distro-slug }} - - name: Create XML Coverage Reports - if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' - run: | - tools --timestamps vm create-xml-coverage-reports ${{ inputs.distro-slug }} - - name: Download Test Run Artifacts id: download-artifacts-from-vm if: always() && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' diff --git a/.github/workflows/test-package-downloads-action-linux.yml b/.github/workflows/test-package-downloads-action-linux.yml index 97dc23ad336..5bc62f53165 100644 --- a/.github/workflows/test-package-downloads-action-linux.yml +++ b/.github/workflows/test-package-downloads-action-linux.yml @@ -178,11 +178,6 @@ jobs: run: | tools --timestamps vm combine-coverage ${{ inputs.distro-slug }} - - name: Create XML Coverage Reports - if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' - run: | - tools --timestamps vm create-xml-coverage-reports ${{ inputs.distro-slug }} - - name: Download Test Run Artifacts id: download-artifacts-from-vm if: always() && steps.spin-up-vm.outcome == 'success' diff --git a/.github/workflows/test-package-downloads-action-macos.yml b/.github/workflows/test-package-downloads-action-macos.yml index 4e66714715a..cab0016181c 100644 --- a/.github/workflows/test-package-downloads-action-macos.yml +++ b/.github/workflows/test-package-downloads-action-macos.yml @@ -171,11 +171,6 @@ jobs: run: | nox --force-color -e combine-coverage - - name: Create XML Coverage Reports - if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled' - run: | - tools --timestamps vm create-xml-coverage-reports ${{ inputs.distro-slug }} - - name: Prepare Test Run Artifacts id: download-artifacts-from-vm if: always() && job.status != 'cancelled' diff --git a/.github/workflows/test-package-downloads-action-windows.yml b/.github/workflows/test-package-downloads-action-windows.yml index 911f9b711a3..fe628957ca4 100644 --- a/.github/workflows/test-package-downloads-action-windows.yml +++ b/.github/workflows/test-package-downloads-action-windows.yml @@ -177,11 +177,6 @@ jobs: run: | tools --timestamps vm combine-coverage ${{ inputs.distro-slug }} - - name: Create XML Coverage Reports - if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' - run: | - tools --timestamps vm create-xml-coverage-reports ${{ inputs.distro-slug }} - - name: Download Test Run Artifacts id: download-artifacts-from-vm if: always() && steps.spin-up-vm.outcome == 'success' From 8d40802c737c258f4fa476bbb68e0f762217ff88 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 30 Sep 2023 08:24:17 +0100 Subject: [PATCH 03/31] This is not a fast test Signed-off-by: Pedro Algarvio --- .../pytests/integration/runners/state/orchestrate/test_events.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytests/integration/runners/state/orchestrate/test_events.py b/tests/pytests/integration/runners/state/orchestrate/test_events.py index 3604d1c4c65..5ebaf500765 100644 --- a/tests/pytests/integration/runners/state/orchestrate/test_events.py +++ b/tests/pytests/integration/runners/state/orchestrate/test_events.py @@ -186,6 +186,7 @@ def test_jid_in_ret_event(salt_run_cli, salt_master, salt_minion, event_listener # This test is flaky on FreeBSD @pytest.mark.skip_on_freebsd +@pytest.mark.slow_test @pytest.mark.skip_on_spawning_platform( reason="The '__low__' global is not populated on spawning platforms" ) From 03a3bccdbf1737d36c752b415e7657e9b8a7579d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 29 Sep 2023 20:38:51 +0100 Subject: [PATCH 04/31] Lock the `pygithub` version being installed in the test Signed-off-by: Pedro Algarvio --- pkg/tests/integration/test_pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tests/integration/test_pip.py b/pkg/tests/integration/test_pip.py index d6102c431e3..a55d8fdd8bb 100644 --- a/pkg/tests/integration/test_pip.py +++ b/pkg/tests/integration/test_pip.py @@ -46,7 +46,7 @@ def test_pip_install(salt_call_cli, install_salt, shell): """ Test pip.install and ensure module can use installed library """ - dep = "PyGithub" + dep = "PyGithub==1.56.0" repo = "https://github.com/saltstack/salt.git" try: From 872293bbb090ea135298751e5d34c47b253ab268 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 30 Sep 2023 14:42:05 +0100 Subject: [PATCH 05/31] Bump lowest coverage version to `6.2` Signed-off-by: Pedro Algarvio --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 16d5ceb5386..1ce1575380c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -337,7 +337,7 @@ def _install_coverage_requirement(session): # plaforms turns the test suite quite slow. # Unit tests don't finish before the 5 hours timeout when they should # finish within 1 to 2 hours. - coverage_requirement = "coverage==5.2" + coverage_requirement = "coverage==6.2" session.install( "--progress-bar=off", coverage_requirement, silent=PIP_INSTALL_SILENT ) From 727c3fe784ff646bf06f16a5cb71091ccee931be Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 29 Sep 2023 13:15:54 -0700 Subject: [PATCH 06/31] Ensure that the correct value of jid_inclue is passed if the argument is included in the passed keyword arguments --- salt/modules/schedule.py | 3 +- tests/pytests/unit/modules/test_schedule.py | 32 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/salt/modules/schedule.py b/salt/modules/schedule.py index 64e2234c3bd..b8553bc6f78 100644 --- a/salt/modules/schedule.py +++ b/salt/modules/schedule.py @@ -521,8 +521,7 @@ def build_schedule_item(name, **kwargs): else: schedule[name]["enabled"] = True - if "jid_include" not in kwargs or kwargs["jid_include"]: - schedule[name]["jid_include"] = True + schedule[name]["jid_include"] = kwargs.get("jid_include", True) if "splay" in kwargs: if isinstance(kwargs["splay"], dict): diff --git a/tests/pytests/unit/modules/test_schedule.py b/tests/pytests/unit/modules/test_schedule.py index d3488559143..d39ae30e5c2 100644 --- a/tests/pytests/unit/modules/test_schedule.py +++ b/tests/pytests/unit/modules/test_schedule.py @@ -198,6 +198,38 @@ def test_build_schedule_item_invalid_jobs_args(): ) == {"comment": comment2, "result": False} +def test_build_schedule_item_jid_include(): + """ + Test build_schedule_item when jid_include is passed and not passed + """ + ret = schedule.build_schedule_item("job1", function="test.args", jid_include=False) + assert ret == { + "function": "test.args", + "maxrunning": 1, + "name": "job1", + "enabled": True, + "jid_include": False, + } + + ret = schedule.build_schedule_item("job1", function="test.args", jid_include=True) + assert ret == { + "function": "test.args", + "maxrunning": 1, + "name": "job1", + "enabled": True, + "jid_include": True, + } + + ret = schedule.build_schedule_item("job1", function="test.args") + assert ret == { + "function": "test.args", + "maxrunning": 1, + "name": "job1", + "enabled": True, + "jid_include": True, + } + + # 'add' function tests: 1 From 6164155e2456f355d541a407cb3e8a919c2b43cd Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 29 Sep 2023 13:17:21 -0700 Subject: [PATCH 07/31] Adding changelog. --- changelog/65302.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/65302.fixed.md diff --git a/changelog/65302.fixed.md b/changelog/65302.fixed.md new file mode 100644 index 00000000000..087bd27314e --- /dev/null +++ b/changelog/65302.fixed.md @@ -0,0 +1 @@ +Ensure that the correct value of jid_inclue is passed if the argument is included in the passed keyword arguments. From a6b3b79923323c0f544a13b22701d42b56855fe7 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sat, 30 Sep 2023 18:31:44 +0100 Subject: [PATCH 08/31] Also check for `SALT_TRANSPORT` in `ci_test_onedir` Signed-off-by: Pedro Algarvio --- noxfile.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/noxfile.py b/noxfile.py index 1ce1575380c..3f6d8aa178d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1174,6 +1174,14 @@ def ci_test_onedir(session): ) ) + transport = os.environ.get("SALT_TRANSPORT") or "zeromq" + valid_transports = ("zeromq", "tcp") + if transport not in valid_transports: + session.error( + "The value for the SALT_TRANSPORT environment variable can only be " + f"one of: {', '.join(valid_transports)}" + ) + _ci_test(session, "zeromq", onedir=True) From cb287ef190b77cd13e294bb0f6030fbdad3d95fc Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 1 Oct 2023 21:13:38 +0100 Subject: [PATCH 09/31] Go back to `coverage==5.5` Signed-off-by: Pedro Algarvio --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 3f6d8aa178d..b914e301721 100644 --- a/noxfile.py +++ b/noxfile.py @@ -337,7 +337,7 @@ def _install_coverage_requirement(session): # plaforms turns the test suite quite slow. # Unit tests don't finish before the 5 hours timeout when they should # finish within 1 to 2 hours. - coverage_requirement = "coverage==6.2" + coverage_requirement = "coverage==5.5" session.install( "--progress-bar=off", coverage_requirement, silent=PIP_INSTALL_SILENT ) From 1405465755d017c5410f8e67b864195c0613e8d3 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 1 Oct 2023 21:19:20 +0100 Subject: [PATCH 10/31] Write directly to `$GITHUB_OUTPUT` Signed-off-by: Pedro Algarvio --- .github/workflows/test-action-macos.yml | 4 +--- .github/workflows/test-action.yml | 4 +--- tools/ci.py | 19 +++++++++++++------ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index 6e4c769953f..ec78982b40e 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -85,9 +85,7 @@ jobs: - name: Generate Test Matrix id: generate-matrix run: | - TEST_MATRIX=$(tools ci matrix ${{ inputs.distro-slug }}) - echo "$TEST_MATRIX" - echo "matrix=$TEST_MATRIX" >> "$GITHUB_OUTPUT" + tools ci matrix ${{ inputs.distro-slug }} test: name: Test diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index da5e6bbe4bc..2d54f22a2f3 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -90,9 +90,7 @@ jobs: - name: Generate Test Matrix id: generate-matrix run: | - TEST_MATRIX=$(tools ci matrix ${{ inputs.distro-slug }}) - echo "$TEST_MATRIX" - echo "matrix=$TEST_MATRIX" >> "$GITHUB_OUTPUT" + tools ci matrix ${{ inputs.distro-slug }} test: name: Test diff --git a/tools/ci.py b/tools/ci.py index 81a18c2d4e9..f45ff409e6a 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -633,7 +633,14 @@ def matrix(ctx: Context, distro_slug: str): if "macos" in distro_slug and chunk == "scenarios": continue _matrix.append({"transport": transport, "tests-chunk": chunk}) - print(json.dumps(_matrix)) + + ctx.info("Generated matrix:") + ctx.print(_matrix, soft_wrap=True) + + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output is not None: + with open(github_output, "a", encoding="utf-8") as wfh: + wfh.write(f"matrix={json.dumps(_matrix)}\n") ctx.exit(0) @@ -667,7 +674,7 @@ def pkg_matrix( ctx.warn("The 'GITHUB_OUTPUT' variable is not set.") if TYPE_CHECKING: assert testing_releases - matrix = [] + _matrix = [] sessions = [ "install", ] @@ -733,20 +740,20 @@ def pkg_matrix( if version < tools.utils.Version("3006.0") ] for version in versions: - matrix.append( + _matrix.append( { "test-chunk": session, "version": version, } ) if distro_slug.startswith("windows"): - matrix[-1]["pkg-type"] = pkg_type.upper() + _matrix[-1]["pkg-type"] = pkg_type.upper() ctx.info("Generated matrix:") - ctx.print(matrix, soft_wrap=True) + ctx.print(_matrix, soft_wrap=True) if github_output is not None: with open(github_output, "a", encoding="utf-8") as wfh: - wfh.write(f"matrix={json.dumps(matrix)}\n") + wfh.write(f"matrix={json.dumps(_matrix)}\n") ctx.exit(0) From 95b43d4226305cb8aed8501f0528c3b74c2f4287 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 2 Oct 2023 07:29:28 +0100 Subject: [PATCH 11/31] Split integration tests into 2 chunks on full test runs Signed-off-by: Pedro Algarvio --- .github/workflows/test-action-macos.yml | 1 - .github/workflows/test-action.yml | 8 ++++---- tests/conftest.py | 6 +++++- tools/ci.py | 18 ++++++++++++++++-- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index ec78982b40e..35699265067 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -324,7 +324,6 @@ jobs: tree -a artifacts if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }} - echo "COVERAGE_FILE=artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}" >> GITHUB_ENV fi - name: Upload Code Coverage Test Run Artifacts diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index 2d54f22a2f3..97da633407c 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -90,7 +90,7 @@ jobs: - name: Generate Test Matrix id: generate-matrix run: | - tools ci matrix ${{ inputs.distro-slug }} + tools ci matrix ${{ fromJSON(inputs.testrun)['type'] == 'full' && '--full ' || '' }}${{ inputs.distro-slug }} test: name: Test @@ -248,7 +248,8 @@ jobs: run: | tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \ --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ (inputs.skip-code-coverage && matrix.tests-chunk != 'unit') && '--skip-code-coverage' || '' }} \ - ${{ inputs.distro-slug }} ${{ matrix.tests-chunk }} -- --slow-tests --core-tests + ${{ inputs.distro-slug }} ${{ matrix.tests-chunk }} -- --slow-tests --core-tests \ + --test-group-count=${{ matrix.test-chunk-total || 1 }} --test-group=${{ matrix.test-chunk-no || 1 }} - name: Combine Coverage Reports if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' @@ -265,8 +266,7 @@ jobs: rm -rf artifacts/salt* tree -a artifacts if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then - mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }} - echo "COVERAGE_FILE=artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}" >> GITHUB_ENV + mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}.${{ matrix.test-chunk-no || '1' }} fi - name: Destroy VM diff --git a/tests/conftest.py b/tests/conftest.py index ff35313df7f..5bb4e5764c3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -835,6 +835,10 @@ def groups_collection_modifyitems(config, items): # We're not selection tests using groups, don't do any filtering return + if group_count == 1: + # Just one group, don't do any filtering + return + total_items = len(items) tests_in_group, deselected = get_group(items, group_count, group_id) @@ -845,7 +849,7 @@ def groups_collection_modifyitems(config, items): terminal_reporter = config.pluginmanager.get_plugin("terminalreporter") terminal_reporter.write( - "Running test group #{} ({} tests)\n".format(group_id, len(items)), + f"Running test group #{group_id}(out of #{group_count}) ({len(items)} tests)\n", yellow=True, ) diff --git a/tools/ci.py b/tools/ci.py index f45ff409e6a..4600da2cc64 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -609,9 +609,12 @@ def define_testrun(ctx: Context, event_name: str, changed_files: pathlib.Path): "distro_slug": { "help": "The distribution slug to generate the matrix for", }, + "full": { + "help": "Full test run", + }, }, ) -def matrix(ctx: Context, distro_slug: str): +def matrix(ctx: Context, distro_slug: str, full: bool = False): """ Generate the test matrix. """ @@ -632,7 +635,18 @@ def matrix(ctx: Context, distro_slug: str): continue if "macos" in distro_slug and chunk == "scenarios": continue - _matrix.append({"transport": transport, "tests-chunk": chunk}) + if full and chunk == "integration": + for idx in range(1, 3): + _matrix.append( + { + "transport": transport, + "tests-chunk": chunk, + "test-chunk-no": idx, + "test-chunk-total": 2, + } + ) + else: + _matrix.append({"transport": transport, "tests-chunk": chunk}) ctx.info("Generated matrix:") ctx.print(_matrix, soft_wrap=True) From 72976d6a0b9ff54ea90a21cadacc3ad0dc1bdd18 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 2 Oct 2023 08:11:02 +0100 Subject: [PATCH 12/31] Typo Signed-off-by: Pedro Algarvio --- tests/support/pytest/transport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/support/pytest/transport.py b/tests/support/pytest/transport.py index eaa8adc8bd4..d71e8fe0254 100644 --- a/tests/support/pytest/transport.py +++ b/tests/support/pytest/transport.py @@ -130,10 +130,10 @@ class Collector(salt.utils.process.SignalHandlingProcess): while True: curr_time = time.time() if time.time() > self.hard_timeout: - log.error("Hard timeout reaced in test collector!") + log.error("Hard timeout reached in test collector!") break if curr_time - last_msg >= self.timeout: - log.error("Receive timeout reaced in test collector!") + log.error("Receive timeout reached in test collector!") break try: payload = yield self._recv() From 18defa2f5db5c563836b6e03dd1cd865b012996e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 2 Oct 2023 11:21:32 +0100 Subject: [PATCH 13/31] Increase splits for faster test results turnaround. Signed-off-by: Pedro Algarvio --- .github/workflows/test-action.yml | 9 +++++---- noxfile.py | 14 ++++++++------ tools/ci.py | 15 +++++++++++---- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index 97da633407c..53724030e46 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -107,6 +107,7 @@ jobs: include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include) }} env: SALT_TRANSPORT: ${{ matrix.transport }} + TEST_GROUP: ${{ matrix.test-group || 1 }} steps: - name: Checkout Source Code @@ -248,8 +249,8 @@ jobs: run: | tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \ --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ (inputs.skip-code-coverage && matrix.tests-chunk != 'unit') && '--skip-code-coverage' || '' }} \ - ${{ inputs.distro-slug }} ${{ matrix.tests-chunk }} -- --slow-tests --core-tests \ - --test-group-count=${{ matrix.test-chunk-total || 1 }} --test-group=${{ matrix.test-chunk-no || 1 }} + -E TEST_GROUP ${{ inputs.distro-slug }} ${{ matrix.tests-chunk }} -- --slow-tests --core-tests \ + --test-group-count=${{ matrix.test-group-count || 1 }} --test-group=${{ matrix.test-group || 1 }} - name: Combine Coverage Reports if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' @@ -266,7 +267,7 @@ jobs: rm -rf artifacts/salt* tree -a artifacts if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then - mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}.${{ matrix.test-chunk-no || '1' }} + mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}.grp${{ matrix.test-group || '1' }} fi - name: Destroy VM @@ -311,7 +312,7 @@ jobs: # always run even if the previous steps fails if: always() && inputs.skip-junit-reports == false with: - check_name: Test Results(${{ inputs.distro-slug }}, transport=${{ matrix.transport }}, tests-chunk=${{ matrix.tests-chunk }}) + check_name: Test Results(${{ inputs.distro-slug }}, transport=${{ matrix.transport }}, tests-chunk=${{ matrix.tests-chunk }}, group=${{ matrix.test-group || '1' }}) report_paths: 'artifacts/xml-unittests-output/*.xml' annotate_only: true diff --git a/noxfile.py b/noxfile.py index b914e301721..313b31a2c18 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1065,6 +1065,8 @@ def _ci_test(session, transport, onedir=False): "scenarios": ["tests/pytests/scenarios"], } + test_group_number = os.environ.get("TEST_GROUP") or "1" + if not session.posargs: chunk_cmd = [] junit_report_filename = "test-results" @@ -1081,20 +1083,20 @@ def _ci_test(session, transport, onedir=False): for values in chunks.values(): for value in values: chunk_cmd.append(f"--ignore={value}") - junit_report_filename = f"test-results-{chunk}" - runtests_log_filename = f"runtests-{chunk}" + junit_report_filename = f"test-results-{chunk}-grp{test_group_number}" + runtests_log_filename = f"runtests-{chunk}-grp{test_group_number}" else: chunk_cmd = chunks[chunk] - junit_report_filename = f"test-results-{chunk}" - runtests_log_filename = f"runtests-{chunk}" + junit_report_filename = f"test-results-{chunk}-grp{test_group_number}" + runtests_log_filename = f"runtests-{chunk}-grp{test_group_number}" if session.posargs: if session.posargs[0] == "--": session.posargs.pop(0) chunk_cmd.extend(session.posargs) else: chunk_cmd = [chunk] + session.posargs - junit_report_filename = "test-results" - runtests_log_filename = "runtests" + junit_report_filename = f"test-results-grp{test_group_number}" + runtests_log_filename = f"runtests-grp{test_group_number}" rerun_failures = os.environ.get("RERUN_FAILURES", "0") == "1" track_code_coverage = os.environ.get("SKIP_CODE_COVERAGE", "0") == "0" diff --git a/tools/ci.py b/tools/ci.py index 4600da2cc64..2bf5af5a25f 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -619,6 +619,12 @@ def matrix(ctx: Context, distro_slug: str, full: bool = False): Generate the test matrix. """ _matrix = [] + _splits = { + "functional": 4, + "integration": 6, + "scenarios": 2, + "unit": 3, + } for transport in ("zeromq", "tcp"): if transport == "tcp": if distro_slug not in ( @@ -635,14 +641,15 @@ def matrix(ctx: Context, distro_slug: str, full: bool = False): continue if "macos" in distro_slug and chunk == "scenarios": continue - if full and chunk == "integration": - for idx in range(1, 3): + splits = _splits.get(chunk) or 1 + if full and splits > 1: + for split in range(1, splits + 1): _matrix.append( { "transport": transport, "tests-chunk": chunk, - "test-chunk-no": idx, - "test-chunk-total": 2, + "test-group": split, + "test-group-count": splits, } ) else: From ef30291e53255e132c174d44d0d27c1789998874 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 2 Oct 2023 19:44:40 +0100 Subject: [PATCH 14/31] Fix code coverage collection/reporting. Create Salt specific coverage report. Signed-off-by: Pedro Algarvio --- .coveragerc | 6 +- .github/workflows/ci.yml | 20 ++++- .github/workflows/nightly.yml | 20 ++++- .github/workflows/scheduled.yml | 20 ++++- .github/workflows/templates/ci.yml.jinja | 20 ++++- noxfile.py | 96 +++++++++++++++++++----- 6 files changed, 146 insertions(+), 36 deletions(-) diff --git a/.coveragerc b/.coveragerc index 40da2c6666d..bdd2587d114 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,11 +7,13 @@ relative_files = True omit = setup.py .nox/* -source = - pkg +source_pkgs = + pkg.tests salt tests tools +disable_warnings = module-not-imported + [report] # Regexes for lines to exclude from consideration diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8165761d85c..b99a0a6ac77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2140,15 +2140,27 @@ jobs: run: | nox --force-color -e combine-coverage - - name: Create Code Coverage HTML Report + - name: Create Salt Code Coverage HTML Report + run: | + nox --force-color -e create-html-coverage-report -- salt + + - name: Upload Salt Code Coverage HTML Report + uses: actions/upload-artifact@v3 + with: + name: code-coverage-salt-html-report + path: artifacts/coverage/html/salt + retention-days: 7 + if-no-files-found: error + + - name: Create Full Code Coverage HTML Report run: | nox --force-color -e create-html-coverage-report - - name: Upload Code Coverage HTML Report + - name: Upload Full Code Coverage HTML Report uses: actions/upload-artifact@v3 with: - name: code-coverage-html-report - path: artifacts/coverage/html + name: code-coverage-full-html-report + path: artifacts/coverage/html/full retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9475cccff58..70dd302031a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2201,15 +2201,27 @@ jobs: run: | nox --force-color -e combine-coverage - - name: Create Code Coverage HTML Report + - name: Create Salt Code Coverage HTML Report + run: | + nox --force-color -e create-html-coverage-report -- salt + + - name: Upload Salt Code Coverage HTML Report + uses: actions/upload-artifact@v3 + with: + name: code-coverage-salt-html-report + path: artifacts/coverage/html/salt + retention-days: 7 + if-no-files-found: error + + - name: Create Full Code Coverage HTML Report run: | nox --force-color -e create-html-coverage-report - - name: Upload Code Coverage HTML Report + - name: Upload Full Code Coverage HTML Report uses: actions/upload-artifact@v3 with: - name: code-coverage-html-report - path: artifacts/coverage/html + name: code-coverage-full-html-report + path: artifacts/coverage/html/full retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 6ac5e68bc52..f8bb593c84c 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -2174,15 +2174,27 @@ jobs: run: | nox --force-color -e combine-coverage - - name: Create Code Coverage HTML Report + - name: Create Salt Code Coverage HTML Report + run: | + nox --force-color -e create-html-coverage-report -- salt + + - name: Upload Salt Code Coverage HTML Report + uses: actions/upload-artifact@v3 + with: + name: code-coverage-salt-html-report + path: artifacts/coverage/html/salt + retention-days: 7 + if-no-files-found: error + + - name: Create Full Code Coverage HTML Report run: | nox --force-color -e create-html-coverage-report - - name: Upload Code Coverage HTML Report + - name: Upload Full Code Coverage HTML Report uses: actions/upload-artifact@v3 with: - name: code-coverage-html-report - path: artifacts/coverage/html + name: code-coverage-full-html-report + path: artifacts/coverage/html/full retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index 67d8d1f877b..e5d0f6171b8 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -355,15 +355,27 @@ run: | nox --force-color -e combine-coverage - - name: Create Code Coverage HTML Report + - name: Create Salt Code Coverage HTML Report + run: | + nox --force-color -e create-html-coverage-report -- salt + + - name: Upload Salt Code Coverage HTML Report + uses: actions/upload-artifact@v3 + with: + name: code-coverage-salt-html-report + path: artifacts/coverage/html/salt + retention-days: 7 + if-no-files-found: error + + - name: Create Full Code Coverage HTML Report run: | nox --force-color -e create-html-coverage-report - - name: Upload Code Coverage HTML Report + - name: Upload Full Code Coverage HTML Report uses: actions/upload-artifact@v3 with: - name: code-coverage-html-report - path: artifacts/coverage/html + name: code-coverage-full-html-report + path: artifacts/coverage/html/full retention-days: 7 if-no-files-found: error <%- endif %> diff --git a/noxfile.py b/noxfile.py index 313b31a2c18..7ebca469e84 100644 --- a/noxfile.py +++ b/noxfile.py @@ -405,7 +405,9 @@ def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): # Always combine and generate the XML coverage report try: session.run( - "coverage", "combine", "--debug=pathmap", env=coverage_base_env + "coverage", + "combine", + env=coverage_base_env, ) except CommandFailed: # Sometimes some of the coverage files are corrupt which would trigger a CommandFailed @@ -417,7 +419,7 @@ def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): "xml", "-o", str(COVERAGE_OUTPUT_DIR.joinpath("tests.xml").relative_to(REPO_ROOT)), - "--omit=salt/*,artifacts/salt/*", + "--omit=salt/*", "--include=tests/*,pkg/tests/*", env=coverage_base_env, ) @@ -428,7 +430,7 @@ def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): "-o", str(COVERAGE_OUTPUT_DIR.joinpath("salt.xml").relative_to(REPO_ROOT)), "--omit=tests/*,pkg/tests/*", - "--include=salt/*,artifacts/salt/*", + "--include=salt/*", env=coverage_base_env, ) # Generate html report for tests code coverage @@ -437,7 +439,7 @@ def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): "html", "-d", str(COVERAGE_OUTPUT_DIR.joinpath("html").relative_to(REPO_ROOT)), - "--omit=salt/*,artifacts/salt/*", + "--omit=salt/*", "--include=tests/*,pkg/tests/*", env=coverage_base_env, ) @@ -448,7 +450,7 @@ def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): "-d", str(COVERAGE_OUTPUT_DIR.joinpath("html").relative_to(REPO_ROOT)), "--omit=tests/*,pkg/tests/*", - "--include=salt/*,artifacts/salt/*", + "--include=salt/*", env=coverage_base_env, ) @@ -499,7 +501,7 @@ def _report_coverage(session): ) cmd_args = [ "--omit=tests/*,pkg/tests/*", - "--include=salt/*,artifacts/salt/*", + "--include=salt/*", ] elif report_section == "tests": @@ -507,7 +509,7 @@ def _report_coverage(session): COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage-tests.json" ) cmd_args = [ - "--omit=salt/*,artifacts/salt/*", + "--omit=salt/*", "--include=tests/*,pkg/tests/*", ] else: @@ -515,9 +517,16 @@ def _report_coverage(session): COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage.json" ) cmd_args = [ - "--include=salt/*,artifacts/salt/*,tests/*,pkg/tests/*", + "--include=salt/*,tests/*,pkg/tests/*", ] + session.run( + "coverage", + "report", + *cmd_args, + env=env, + ) + session.run( "coverage", "json", @@ -526,12 +535,6 @@ def _report_coverage(session): *cmd_args, env=env, ) - session.run( - "coverage", - "report", - *cmd_args, - env=env, - ) @nox.session(python=_PYTHON_VERSIONS, name="test-parametrized") @@ -1379,14 +1382,71 @@ def create_html_coverage_report(session): "COVERAGE_FILE": str(COVERAGE_FILE), } + report_section = None + if session.posargs: + report_section = session.posargs.pop(0) + if report_section not in ("salt", "tests"): + session.error("The report section can only be one of 'salt', 'tests'.") + if session.posargs: + session.error( + "Only one argument can be passed to the session, which is optional " + "and is one of 'salt', 'tests'." + ) + + if not IS_WINDOWS: + # The coverage file might have come from a windows machine, fix paths + with sqlite3.connect(COVERAGE_FILE) as db: + res = db.execute(r"SELECT * FROM file WHERE path LIKE '%salt\%'") + if res.fetchone(): + session_warn( + session, + "Replacing backwards slashes with forward slashes on file " + "paths in the coverage database", + ) + db.execute(r"UPDATE OR IGNORE file SET path=replace(path, '\', '/');") + + if report_section == "salt": + report_dir = str( + COVERAGE_OUTPUT_DIR.joinpath("html", "salt").relative_to(REPO_ROOT) + ) + json_coverage_file = ( + COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage-salt.json" + ) + cmd_args = [ + "--omit=tests/*,pkg/tests/*", + "--include=salt/*", + ] + + elif report_section == "tests": + report_dir = str( + COVERAGE_OUTPUT_DIR.joinpath("html", "tests").relative_to(REPO_ROOT) + ) + json_coverage_file = ( + COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage-tests.json" + ) + cmd_args = [ + "--omit=salt/*", + "--include=tests/*,pkg/tests/*", + ] + else: + report_dir = str( + COVERAGE_OUTPUT_DIR.joinpath("html", "full").relative_to(REPO_ROOT) + ) + json_coverage_file = ( + COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage.json" + ) + cmd_args = [ + "--include=salt/*,tests/*,pkg/tests/*", + ] + # Generate html report for Salt and tests combined code coverage session.run( "coverage", "html", "-d", - str(COVERAGE_OUTPUT_DIR.joinpath("html").relative_to(REPO_ROOT)), - "--include=salt/*,artifacts/salt/*,tests/*,pkg/tests/*", + report_dir, "--show-contexts", + *cmd_args, env=env, ) @@ -1406,7 +1466,7 @@ def _create_xml_coverage_reports(session): "xml", "-o", str(COVERAGE_OUTPUT_DIR.joinpath("tests.xml").relative_to(REPO_ROOT)), - "--omit=salt/*,artifacts/salt/*", + "--omit=salt/*", "--include=tests/*,pkg/tests/*", env=env, ) @@ -1421,7 +1481,7 @@ def _create_xml_coverage_reports(session): "-o", str(COVERAGE_OUTPUT_DIR.joinpath("salt.xml").relative_to(REPO_ROOT)), "--omit=tests/*,pkg/tests/*", - "--include=salt/*,artifacts/salt/*", + "--include=salt/*", env=env, ) except CommandFailed: From f1e138731b8314d7a1acd5305267c4e4a0bb164c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 2 Oct 2023 12:20:32 +0100 Subject: [PATCH 15/31] Split build deps onedir jobs Signed-off-by: Pedro Algarvio --- .github/actions/build-onedir-deps/action.yml | 16 +- .github/workflows/build-deps-onedir-linux.yml | 81 ++++++++ .github/workflows/build-deps-onedir-macos.yml | 83 +++++++++ .../workflows/build-deps-onedir-windows.yml | 84 +++++++++ .github/workflows/build-deps-onedir.yml | 174 ------------------ .github/workflows/ci.yml | 40 +++- .github/workflows/nightly.yml | 40 +++- .github/workflows/scheduled.yml | 40 +++- .github/workflows/staging.yml | 40 +++- .github/workflows/templates/ci.yml.jinja | 13 +- 10 files changed, 410 insertions(+), 201 deletions(-) create mode 100644 .github/workflows/build-deps-onedir-linux.yml create mode 100644 .github/workflows/build-deps-onedir-macos.yml create mode 100644 .github/workflows/build-deps-onedir-windows.yml delete mode 100644 .github/workflows/build-deps-onedir.yml diff --git a/.github/actions/build-onedir-deps/action.yml b/.github/actions/build-onedir-deps/action.yml index fceb79f0eb7..50e302bd40f 100644 --- a/.github/actions/build-onedir-deps/action.yml +++ b/.github/actions/build-onedir-deps/action.yml @@ -43,13 +43,15 @@ runs: with: path: artifacts/${{ inputs.package-name }} key: > - ${{ inputs.cache-prefix }}|${{ inputs.python-version }}|deps|${{ inputs.platform }}|${{ inputs.arch }}|${{ inputs.package-name }}| - ${{ hashFiles( - format('{0}/.relenv/**/*.xz', github.workspace), - 'requirements/static/pkg/*/*.txt', - '.github/actions/build-onedir-deps/action.yml', - 'cicd/shared-gh-workflows-context.yml' - ) }} + ${{ inputs.cache-prefix }}|${{ inputs.python-version }}|deps|${{ inputs.platform }}|${{ inputs.arch }}|${{ inputs.package-name }}|${{ + hashFiles( + format('{0}/.relenv/**/*.xz', github.workspace), + 'requirements/static/pkg/*/*.txt', + '.github/actions/build-onedir-deps/action.yml', + '.github/workflows/build-deps-onedir-*.yml', + 'cicd/shared-gh-workflows-context.yml' + ) + }} - name: Install Salt Onedir Package Dependencies shell: bash diff --git a/.github/workflows/build-deps-onedir-linux.yml b/.github/workflows/build-deps-onedir-linux.yml new file mode 100644 index 00000000000..ebd686defdf --- /dev/null +++ b/.github/workflows/build-deps-onedir-linux.yml @@ -0,0 +1,81 @@ +--- +name: Build Packaging Dependencies Onedir + +on: + workflow_call: + inputs: + salt-version: + type: string + required: true + description: The Salt version to set prior to building packages. + github-hosted-runners: + type: boolean + required: true + self-hosted-runners: + type: boolean + required: true + cache-seed: + required: true + type: string + description: Seed used to invalidate caches + relenv-version: + required: true + type: string + description: The version of relenv to use + python-version: + required: true + type: string + description: The version of python to use with relenv + +env: + RELENV_DATA: "${{ github.workspace }}/.relenv" + COLUMNS: 190 + AWS_MAX_ATTEMPTS: "10" + AWS_RETRY_MODE: "adaptive" + PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ + PIP_EXTRA_INDEX_URL: https://pypi.org/simple + +jobs: + + build-deps-linux: + name: Linux + if: ${{ inputs.self-hosted-runners }} + strategy: + fail-fast: false + matrix: + arch: + - x86_64 + - aarch64 + runs-on: + - self-hosted + - linux + - ${{ matrix.arch }} + steps: + + - name: "Throttle Builds" + shell: bash + run: | + t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" + + - uses: actions/checkout@v4 + + - name: Setup Relenv + id: setup-relenv + uses: ./.github/actions/setup-relenv + with: + platform: linux + arch: ${{ matrix.arch }} + version: ${{ inputs.relenv-version }} + cache-seed: ${{ inputs.cache-seed }} + python-version: ${{ inputs.python-version }} + + - name: Setup Python Tools Scripts + uses: ./.github/actions/setup-python-tools-scripts + + - name: Install Salt Packaging Dependencies into Relenv Onedir + uses: ./.github/actions/build-onedir-deps + with: + platform: linux + arch: ${{ matrix.arch }} + python-version: "${{ inputs.python-version }}" + cache-prefix: ${{ inputs.cache-seed }}|relenv|${{ steps.setup-relenv.outputs.version }} diff --git a/.github/workflows/build-deps-onedir-macos.yml b/.github/workflows/build-deps-onedir-macos.yml new file mode 100644 index 00000000000..033a650d0a4 --- /dev/null +++ b/.github/workflows/build-deps-onedir-macos.yml @@ -0,0 +1,83 @@ +--- +name: Build Packaging Dependencies Onedir + +on: + workflow_call: + inputs: + salt-version: + type: string + required: true + description: The Salt version to set prior to building packages. + github-hosted-runners: + type: boolean + required: true + self-hosted-runners: + type: boolean + required: true + cache-seed: + required: true + type: string + description: Seed used to invalidate caches + relenv-version: + required: true + type: string + description: The version of relenv to use + python-version: + required: true + type: string + description: The version of python to use with relenv + +env: + RELENV_DATA: "${{ github.workspace }}/.relenv" + COLUMNS: 190 + AWS_MAX_ATTEMPTS: "10" + AWS_RETRY_MODE: "adaptive" + PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ + PIP_EXTRA_INDEX_URL: https://pypi.org/simple + +jobs: + + build-deps-macos: + name: macOS + if: ${{ inputs.github-hosted-runners }} + strategy: + fail-fast: false + max-parallel: 2 + matrix: + arch: + - x86_64 + runs-on: macos-12 + steps: + + - name: "Throttle Builds" + shell: bash + run: | + t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" + + - uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Setup Relenv + id: setup-relenv + uses: ./.github/actions/setup-relenv + with: + platform: darwin + arch: ${{ matrix.arch }} + version: ${{ inputs.relenv-version }} + cache-seed: ${{ inputs.cache-seed }} + python-version: ${{ inputs.python-version }} + + - name: Setup Python Tools Scripts + uses: ./.github/actions/setup-python-tools-scripts + + - name: Install Salt Packaging Dependencies into Relenv Onedir + uses: ./.github/actions/build-onedir-deps + with: + platform: darwin + arch: ${{ matrix.arch }} + python-version: "${{ inputs.python-version }}" + cache-prefix: ${{ inputs.cache-seed }}|relenv|${{ steps.setup-relenv.outputs.version }} diff --git a/.github/workflows/build-deps-onedir-windows.yml b/.github/workflows/build-deps-onedir-windows.yml new file mode 100644 index 00000000000..bb7538a6ef0 --- /dev/null +++ b/.github/workflows/build-deps-onedir-windows.yml @@ -0,0 +1,84 @@ +--- +name: Build Packaging Dependencies Onedir + +on: + workflow_call: + inputs: + salt-version: + type: string + required: true + description: The Salt version to set prior to building packages. + github-hosted-runners: + type: boolean + required: true + self-hosted-runners: + type: boolean + required: true + cache-seed: + required: true + type: string + description: Seed used to invalidate caches + relenv-version: + required: true + type: string + description: The version of relenv to use + python-version: + required: true + type: string + description: The version of python to use with relenv + +env: + RELENV_DATA: "${{ github.workspace }}/.relenv" + COLUMNS: 190 + AWS_MAX_ATTEMPTS: "10" + AWS_RETRY_MODE: "adaptive" + PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ + PIP_EXTRA_INDEX_URL: https://pypi.org/simple + +jobs: + + build-deps-windows: + name: Windows + if: ${{ inputs.github-hosted-runners }} + strategy: + fail-fast: false + max-parallel: 2 + matrix: + arch: + - x86 + - amd64 + runs-on: windows-latest + steps: + + - name: "Throttle Builds" + shell: bash + run: | + t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" + + - uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Setup Relenv + id: setup-relenv + uses: ./.github/actions/setup-relenv + with: + platform: windows + arch: ${{ matrix.arch }} + version: ${{ inputs.relenv-version }} + cache-seed: ${{ inputs.cache-seed }} + python-version: ${{ inputs.python-version }} + + - name: Setup Python Tools Scripts + uses: ./.github/actions/setup-python-tools-scripts + + - name: Install Salt Packaging Dependencies into Relenv Onedir + uses: ./.github/actions/build-onedir-deps + with: + platform: windows + arch: ${{ matrix.arch }} + python-version: "${{ inputs.python-version }}" + cache-prefix: ${{ inputs.cache-seed }}|relenv|${{ steps.setup-relenv.outputs.version }} diff --git a/.github/workflows/build-deps-onedir.yml b/.github/workflows/build-deps-onedir.yml deleted file mode 100644 index 3cf048bd4d0..00000000000 --- a/.github/workflows/build-deps-onedir.yml +++ /dev/null @@ -1,174 +0,0 @@ ---- -name: Build Packaging Dependencies Onedir - -on: - workflow_call: - inputs: - salt-version: - type: string - required: true - description: The Salt version to set prior to building packages. - github-hosted-runners: - type: boolean - required: true - self-hosted-runners: - type: boolean - required: true - cache-seed: - required: true - type: string - description: Seed used to invalidate caches - relenv-version: - required: true - type: string - description: The version of relenv to use - python-version: - required: true - type: string - description: The version of python to use with relenv - -env: - RELENV_DATA: "${{ github.workspace }}/.relenv" - COLUMNS: 190 - AWS_MAX_ATTEMPTS: "10" - AWS_RETRY_MODE: "adaptive" - PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ - PIP_EXTRA_INDEX_URL: https://pypi.org/simple - -jobs: - - build-deps-linux: - name: Linux - if: ${{ inputs.self-hosted-runners }} - strategy: - fail-fast: false - matrix: - arch: - - x86_64 - - aarch64 - runs-on: - - self-hosted - - linux - - ${{ matrix.arch }} - steps: - - - name: "Throttle Builds" - shell: bash - run: | - t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" - - - uses: actions/checkout@v4 - - - name: Setup Relenv - id: setup-relenv - uses: ./.github/actions/setup-relenv - with: - platform: linux - arch: ${{ matrix.arch }} - version: ${{ inputs.relenv-version }} - cache-seed: ${{ inputs.cache-seed }} - python-version: ${{ inputs.python-version }} - - - name: Setup Python Tools Scripts - uses: ./.github/actions/setup-python-tools-scripts - - - name: Install Salt Packaging Dependencies into Relenv Onedir - uses: ./.github/actions/build-onedir-deps - with: - platform: linux - arch: ${{ matrix.arch }} - python-version: "${{ inputs.python-version }}" - cache-prefix: ${{ inputs.cache-seed }}|relenv|${{ steps.setup-relenv.outputs.version }} - - - build-deps-windows: - name: Windows - if: ${{ inputs.github-hosted-runners }} - strategy: - fail-fast: false - max-parallel: 2 - matrix: - arch: - - x86 - - amd64 - runs-on: windows-latest - steps: - - - name: "Throttle Builds" - shell: bash - run: | - t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" - - - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Setup Relenv - id: setup-relenv - uses: ./.github/actions/setup-relenv - with: - platform: windows - arch: ${{ matrix.arch }} - version: ${{ inputs.relenv-version }} - cache-seed: ${{ inputs.cache-seed }} - python-version: ${{ inputs.python-version }} - - - name: Setup Python Tools Scripts - uses: ./.github/actions/setup-python-tools-scripts - - - name: Install Salt Packaging Dependencies into Relenv Onedir - uses: ./.github/actions/build-onedir-deps - with: - platform: windows - arch: ${{ matrix.arch }} - python-version: "${{ inputs.python-version }}" - cache-prefix: ${{ inputs.cache-seed }}|relenv|${{ steps.setup-relenv.outputs.version }} - - - build-deps-macos: - name: macOS - if: ${{ inputs.github-hosted-runners }} - strategy: - fail-fast: false - max-parallel: 2 - matrix: - arch: - - x86_64 - runs-on: macos-12 - steps: - - - name: "Throttle Builds" - shell: bash - run: | - t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" - - - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Setup Relenv - id: setup-relenv - uses: ./.github/actions/setup-relenv - with: - platform: darwin - arch: ${{ matrix.arch }} - version: ${{ inputs.relenv-version }} - cache-seed: ${{ inputs.cache-seed }} - python-version: ${{ inputs.python-version }} - - - name: Setup Python Tools Scripts - uses: ./.github/actions/setup-python-tools-scripts - - - name: Install Salt Packaging Dependencies into Relenv Onedir - uses: ./.github/actions/build-onedir-deps - with: - platform: darwin - arch: ${{ matrix.arch }} - python-version: "${{ inputs.python-version }}" - cache-prefix: ${{ inputs.cache-seed }}|relenv|${{ steps.setup-relenv.outputs.version }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b99a0a6ac77..2c2fe347233 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -431,12 +431,40 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - build-deps-onedir: + build-deps-onedir-linux: name: Build Dependencies Onedir if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-deps-onedir'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - uses: ./.github/workflows/build-deps-onedir.yml + uses: ./.github/workflows/build-deps-onedir-linux.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-deps-onedir-windows: + name: Build Dependencies Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-deps-onedir'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + needs: + - prepare-workflow + uses: ./.github/workflows/build-deps-onedir-windows.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-deps-onedir-macos: + name: Build Dependencies Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-deps-onedir'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + needs: + - prepare-workflow + uses: ./.github/workflows/build-deps-onedir-macos.yml with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -450,7 +478,9 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} needs: - prepare-workflow - - build-deps-onedir + - build-deps-onedir-linux + - build-deps-onedir-windows + - build-deps-onedir-macos - build-source-tarball uses: ./.github/workflows/build-salt-onedir.yml with: @@ -2175,7 +2205,9 @@ jobs: - pre-commit - lint - build-docs - - build-deps-onedir + - build-deps-onedir-linux + - build-deps-onedir-windows + - build-deps-onedir-macos - build-salt-onedir - build-rpm-pkgs-src - build-deb-pkgs-src diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 70dd302031a..5c52d2717fe 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -480,12 +480,40 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - build-deps-onedir: + build-deps-onedir-linux: name: Build Dependencies Onedir if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-deps-onedir'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - uses: ./.github/workflows/build-deps-onedir.yml + uses: ./.github/workflows/build-deps-onedir-linux.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-deps-onedir-windows: + name: Build Dependencies Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-deps-onedir'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + needs: + - prepare-workflow + uses: ./.github/workflows/build-deps-onedir-windows.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-deps-onedir-macos: + name: Build Dependencies Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-deps-onedir'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + needs: + - prepare-workflow + uses: ./.github/workflows/build-deps-onedir-macos.yml with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -499,7 +527,9 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} needs: - prepare-workflow - - build-deps-onedir + - build-deps-onedir-linux + - build-deps-onedir-windows + - build-deps-onedir-macos - build-source-tarball uses: ./.github/workflows/build-salt-onedir.yml with: @@ -2975,7 +3005,9 @@ jobs: - pre-commit - lint - build-docs - - build-deps-onedir + - build-deps-onedir-linux + - build-deps-onedir-windows + - build-deps-onedir-macos - build-salt-onedir - build-rpm-pkgs-src - build-deb-pkgs-src diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index f8bb593c84c..85f15461dc3 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -465,12 +465,40 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - build-deps-onedir: + build-deps-onedir-linux: name: Build Dependencies Onedir if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-deps-onedir'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - uses: ./.github/workflows/build-deps-onedir.yml + uses: ./.github/workflows/build-deps-onedir-linux.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-deps-onedir-windows: + name: Build Dependencies Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-deps-onedir'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + needs: + - prepare-workflow + uses: ./.github/workflows/build-deps-onedir-windows.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-deps-onedir-macos: + name: Build Dependencies Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-deps-onedir'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + needs: + - prepare-workflow + uses: ./.github/workflows/build-deps-onedir-macos.yml with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -484,7 +512,9 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} needs: - prepare-workflow - - build-deps-onedir + - build-deps-onedir-linux + - build-deps-onedir-windows + - build-deps-onedir-macos - build-source-tarball uses: ./.github/workflows/build-salt-onedir.yml with: @@ -2211,7 +2241,9 @@ jobs: - pre-commit - lint - build-docs - - build-deps-onedir + - build-deps-onedir-linux + - build-deps-onedir-windows + - build-deps-onedir-macos - build-salt-onedir - build-rpm-pkgs-src - build-deb-pkgs-src diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index d2daa5abdfd..416940e344c 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -475,12 +475,40 @@ jobs: with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" - build-deps-onedir: + build-deps-onedir-linux: name: Build Dependencies Onedir if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-deps-onedir'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - uses: ./.github/workflows/build-deps-onedir.yml + uses: ./.github/workflows/build-deps-onedir-linux.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-deps-onedir-windows: + name: Build Dependencies Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-deps-onedir'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + needs: + - prepare-workflow + uses: ./.github/workflows/build-deps-onedir-windows.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-deps-onedir-macos: + name: Build Dependencies Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-deps-onedir'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + needs: + - prepare-workflow + uses: ./.github/workflows/build-deps-onedir-macos.yml with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -494,7 +522,9 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} needs: - prepare-workflow - - build-deps-onedir + - build-deps-onedir-linux + - build-deps-onedir-windows + - build-deps-onedir-macos - build-source-tarball uses: ./.github/workflows/build-salt-onedir.yml with: @@ -3755,7 +3785,9 @@ jobs: - pre-commit - lint - build-docs - - build-deps-onedir + - build-deps-onedir-linux + - build-deps-onedir-windows + - build-deps-onedir-macos - build-salt-onedir - build-rpm-pkgs-src - build-deb-pkgs-src diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index e5d0f6171b8..896bc880354 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -254,14 +254,16 @@ <%- set job_name = "build-deps-onedir" %> <%- if includes.get(job_name, True) %> + <%- for platform in ("linux", "windows", "macos") %> + <%- set platform_job_name = "{}-{}".format(job_name, platform) %> - <{ job_name }>: - <%- do conclusion_needs.append(job_name) %> + <{ platform_job_name }>: + <%- do conclusion_needs.append(platform_job_name) %> name: Build Dependencies Onedir if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['<{ job_name }>'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - uses: ./.github/workflows/build-deps-onedir.yml + uses: ./.github/workflows/build-deps-onedir-<{ platform }>.yml with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -270,6 +272,7 @@ relenv-version: "<{ relenv_version }>" python-version: "<{ python_version }>" + <%- endfor %> <%- endif %> @@ -282,7 +285,9 @@ if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['<{ job_name }>'] }} needs: - prepare-workflow - - build-deps-onedir + <%- for platform in ("linux", "windows", "macos") %> + - build-deps-onedir-<{ platform }> + <%- endfor %> - build-source-tarball uses: ./.github/workflows/build-salt-onedir.yml with: From 5be7c858eb97652b43f5e984d9d01325782d3751 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 2 Oct 2023 12:21:19 +0100 Subject: [PATCH 16/31] Split build salt onedir jobs Signed-off-by: Pedro Algarvio --- .github/actions/build-onedir-salt/action.yml | 16 +- .github/workflows/build-salt-onedir-linux.yml | 88 ++++++++ .github/workflows/build-salt-onedir-macos.yml | 89 ++++++++ .../workflows/build-salt-onedir-windows.yml | 91 ++++++++ .github/workflows/build-salt-onedir.yml | 194 ------------------ .github/workflows/ci.yml | 118 +++++++---- .github/workflows/nightly.yml | 120 +++++++---- .github/workflows/scheduled.yml | 118 +++++++---- .github/workflows/staging.yml | 120 +++++++---- .../templates/build-ci-deps.yml.jinja | 6 +- .../templates/build-packages.yml.jinja | 14 +- .../workflows/templates/build-repos.yml.jinja | 20 +- .github/workflows/templates/ci.yml.jinja | 11 +- 13 files changed, 603 insertions(+), 402 deletions(-) create mode 100644 .github/workflows/build-salt-onedir-linux.yml create mode 100644 .github/workflows/build-salt-onedir-macos.yml create mode 100644 .github/workflows/build-salt-onedir-windows.yml delete mode 100644 .github/workflows/build-salt-onedir.yml diff --git a/.github/actions/build-onedir-salt/action.yml b/.github/actions/build-onedir-salt/action.yml index 1f115607e81..10e92bbcb89 100644 --- a/.github/actions/build-onedir-salt/action.yml +++ b/.github/actions/build-onedir-salt/action.yml @@ -47,13 +47,15 @@ runs: with: path: artifacts/${{ inputs.package-name }} key: > - ${{ inputs.cache-prefix }}|${{ inputs.python-version }}|deps|${{ inputs.platform }}|${{ inputs.arch }}|${{ inputs.package-name }}| - ${{ hashFiles( - format('{0}/.relenv/**/*.xz', github.workspace), - 'requirements/static/pkg/*/*.txt', - '.github/actions/build-onedir-deps/action.yml', - 'cicd/shared-gh-workflows-context.yml' - ) }} + ${{ inputs.cache-prefix }}|${{ inputs.python-version }}|deps|${{ inputs.platform }}|${{ inputs.arch }}|${{ inputs.package-name }}|${{ + hashFiles( + format('{0}/.relenv/**/*.xz', github.workspace), + 'requirements/static/pkg/*/*.txt', + '.github/actions/build-onedir-deps/action.yml', + '.github/workflows/build-deps-onedir-*.yml', + 'cicd/shared-gh-workflows-context.yml' + ) + }} - name: Download Source Tarball uses: actions/download-artifact@v3 diff --git a/.github/workflows/build-salt-onedir-linux.yml b/.github/workflows/build-salt-onedir-linux.yml new file mode 100644 index 00000000000..2550ece63a9 --- /dev/null +++ b/.github/workflows/build-salt-onedir-linux.yml @@ -0,0 +1,88 @@ +--- +name: Build Salt Onedir + +on: + workflow_call: + inputs: + salt-version: + type: string + required: true + description: The Salt version to set prior to building packages. + github-hosted-runners: + type: boolean + required: true + self-hosted-runners: + type: boolean + required: true + cache-seed: + required: true + type: string + description: Seed used to invalidate caches + relenv-version: + required: true + type: string + description: The version of relenv to use + python-version: + required: true + type: string + description: The version of python to use with relenv + +env: + RELENV_DATA: "${{ github.workspace }}/.relenv" + COLUMNS: 190 + AWS_MAX_ATTEMPTS: "10" + AWS_RETRY_MODE: "adaptive" + PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ + PIP_EXTRA_INDEX_URL: https://pypi.org/simple + +jobs: + + build-salt-linux: + name: Linux + if: ${{ inputs.self-hosted-runners }} + strategy: + fail-fast: false + matrix: + arch: + - x86_64 + - aarch64 + runs-on: + - self-hosted + - linux + - ${{ matrix.arch }} + steps: + + - name: "Throttle Builds" + shell: bash + run: | + t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" + + - uses: actions/checkout@v4 + + - name: Setup Python Tools Scripts + uses: ./.github/actions/setup-python-tools-scripts + + - name: Setup Salt Version + id: setup-salt-version + uses: ./.github/actions/setup-salt-version + with: + salt-version: "${{ inputs.salt-version }}" + + - name: Setup Relenv + uses: ./.github/actions/setup-relenv + id: setup-relenv + with: + platform: linux + arch: ${{ matrix.arch }} + version: ${{ inputs.relenv-version }} + cache-seed: ${{ inputs.cache-seed }} + python-version: ${{ inputs.python-version }} + + - name: Install Salt into Relenv Onedir + uses: ./.github/actions/build-onedir-salt + with: + platform: linux + arch: ${{ matrix.arch }} + salt-version: "${{ inputs.salt-version }}" + python-version: "${{ inputs.python-version }}" + cache-prefix: ${{ inputs.cache-seed }}|relenv|${{ steps.setup-relenv.outputs.version }} diff --git a/.github/workflows/build-salt-onedir-macos.yml b/.github/workflows/build-salt-onedir-macos.yml new file mode 100644 index 00000000000..eedae19305c --- /dev/null +++ b/.github/workflows/build-salt-onedir-macos.yml @@ -0,0 +1,89 @@ +--- +name: Build Salt Onedir + +on: + workflow_call: + inputs: + salt-version: + type: string + required: true + description: The Salt version to set prior to building packages. + github-hosted-runners: + type: boolean + required: true + self-hosted-runners: + type: boolean + required: true + cache-seed: + required: true + type: string + description: Seed used to invalidate caches + relenv-version: + required: true + type: string + description: The version of relenv to use + python-version: + required: true + type: string + description: The version of python to use with relenv + +env: + RELENV_DATA: "${{ github.workspace }}/.relenv" + COLUMNS: 190 + AWS_MAX_ATTEMPTS: "10" + AWS_RETRY_MODE: "adaptive" + PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ + PIP_EXTRA_INDEX_URL: https://pypi.org/simple + +jobs: + + build-salt-macos: + name: macOS + if: ${{ inputs.github-hosted-runners }} + strategy: + fail-fast: false + max-parallel: 2 + matrix: + arch: + - x86_64 + runs-on: macos-12 + steps: + + - name: "Throttle Builds" + shell: bash + run: | + t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" + + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Setup Relenv + id: setup-relenv + uses: ./.github/actions/setup-relenv + with: + platform: darwin + arch: ${{ matrix.arch }} + version: ${{ inputs.relenv-version }} + cache-seed: ${{ inputs.cache-seed }} + python-version: ${{ inputs.python-version }} + + - name: Setup Python Tools Scripts + uses: ./.github/actions/setup-python-tools-scripts + + - name: Setup Salt Version + id: setup-salt-version + uses: ./.github/actions/setup-salt-version + with: + salt-version: "${{ inputs.salt-version }}" + + - name: Install Salt into Relenv Onedir + uses: ./.github/actions/build-onedir-salt + with: + platform: darwin + arch: ${{ matrix.arch }} + salt-version: "${{ inputs.salt-version }}" + python-version: "${{ inputs.python-version }}" + cache-prefix: ${{ inputs.cache-seed }}|relenv|${{ steps.setup-relenv.outputs.version }} diff --git a/.github/workflows/build-salt-onedir-windows.yml b/.github/workflows/build-salt-onedir-windows.yml new file mode 100644 index 00000000000..6ae148f9122 --- /dev/null +++ b/.github/workflows/build-salt-onedir-windows.yml @@ -0,0 +1,91 @@ +--- +name: Build Salt Onedir + +on: + workflow_call: + inputs: + salt-version: + type: string + required: true + description: The Salt version to set prior to building packages. + github-hosted-runners: + type: boolean + required: true + self-hosted-runners: + type: boolean + required: true + cache-seed: + required: true + type: string + description: Seed used to invalidate caches + relenv-version: + required: true + type: string + description: The version of relenv to use + python-version: + required: true + type: string + description: The version of python to use with relenv + +env: + RELENV_DATA: "${{ github.workspace }}/.relenv" + COLUMNS: 190 + AWS_MAX_ATTEMPTS: "10" + AWS_RETRY_MODE: "adaptive" + PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ + PIP_EXTRA_INDEX_URL: https://pypi.org/simple + +jobs: + + build-salt-windows: + name: Windows + if: ${{ inputs.github-hosted-runners }} + strategy: + fail-fast: false + max-parallel: 2 + matrix: + arch: + - x86 + - amd64 + runs-on: windows-latest + steps: + + - name: "Throttle Builds" + shell: bash + run: | + t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" + + - uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Setup Relenv + id: setup-relenv + uses: ./.github/actions/setup-relenv + with: + platform: windows + arch: ${{ matrix.arch }} + version: ${{ inputs.relenv-version }} + cache-seed: ${{ inputs.cache-seed }} + python-version: ${{ inputs.python-version }} + + - name: Setup Python Tools Scripts + uses: ./.github/actions/setup-python-tools-scripts + + - name: Setup Salt Version + id: setup-salt-version + uses: ./.github/actions/setup-salt-version + with: + salt-version: "${{ inputs.salt-version }}" + + - name: Install Salt into Relenv Onedir + uses: ./.github/actions/build-onedir-salt + with: + platform: windows + arch: ${{ matrix.arch }} + salt-version: "${{ inputs.salt-version }}" + python-version: "${{ inputs.python-version }}" + cache-prefix: ${{ inputs.cache-seed }}|relenv|${{ steps.setup-relenv.outputs.version }} diff --git a/.github/workflows/build-salt-onedir.yml b/.github/workflows/build-salt-onedir.yml deleted file mode 100644 index 6ee7e802be0..00000000000 --- a/.github/workflows/build-salt-onedir.yml +++ /dev/null @@ -1,194 +0,0 @@ ---- -name: Build Salt Onedir - -on: - workflow_call: - inputs: - salt-version: - type: string - required: true - description: The Salt version to set prior to building packages. - github-hosted-runners: - type: boolean - required: true - self-hosted-runners: - type: boolean - required: true - cache-seed: - required: true - type: string - description: Seed used to invalidate caches - relenv-version: - required: true - type: string - description: The version of relenv to use - python-version: - required: true - type: string - description: The version of python to use with relenv - -env: - RELENV_DATA: "${{ github.workspace }}/.relenv" - COLUMNS: 190 - AWS_MAX_ATTEMPTS: "10" - AWS_RETRY_MODE: "adaptive" - PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/ - PIP_EXTRA_INDEX_URL: https://pypi.org/simple - -jobs: - - build-salt-linux: - name: Linux - if: ${{ inputs.self-hosted-runners }} - strategy: - fail-fast: false - matrix: - arch: - - x86_64 - - aarch64 - runs-on: - - self-hosted - - linux - - ${{ matrix.arch }} - steps: - - - name: "Throttle Builds" - shell: bash - run: | - t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" - - - uses: actions/checkout@v4 - - - name: Setup Python Tools Scripts - uses: ./.github/actions/setup-python-tools-scripts - - - name: Setup Salt Version - id: setup-salt-version - uses: ./.github/actions/setup-salt-version - with: - salt-version: "${{ inputs.salt-version }}" - - - name: Setup Relenv - uses: ./.github/actions/setup-relenv - id: setup-relenv - with: - platform: linux - arch: ${{ matrix.arch }} - version: ${{ inputs.relenv-version }} - cache-seed: ${{ inputs.cache-seed }} - python-version: ${{ inputs.python-version }} - - - name: Install Salt into Relenv Onedir - uses: ./.github/actions/build-onedir-salt - with: - platform: linux - arch: ${{ matrix.arch }} - salt-version: "${{ inputs.salt-version }}" - python-version: "${{ inputs.python-version }}" - cache-prefix: ${{ inputs.cache-seed }}|relenv|${{ steps.setup-relenv.outputs.version }} - - - build-salt-windows: - name: Windows - if: ${{ inputs.github-hosted-runners }} - strategy: - fail-fast: false - max-parallel: 2 - matrix: - arch: - - x86 - - amd64 - runs-on: windows-latest - steps: - - - name: "Throttle Builds" - shell: bash - run: | - t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" - - - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Setup Relenv - id: setup-relenv - uses: ./.github/actions/setup-relenv - with: - platform: windows - arch: ${{ matrix.arch }} - version: ${{ inputs.relenv-version }} - cache-seed: ${{ inputs.cache-seed }} - python-version: ${{ inputs.python-version }} - - - name: Setup Python Tools Scripts - uses: ./.github/actions/setup-python-tools-scripts - - - name: Setup Salt Version - id: setup-salt-version - uses: ./.github/actions/setup-salt-version - with: - salt-version: "${{ inputs.salt-version }}" - - - name: Install Salt into Relenv Onedir - uses: ./.github/actions/build-onedir-salt - with: - platform: windows - arch: ${{ matrix.arch }} - salt-version: "${{ inputs.salt-version }}" - python-version: "${{ inputs.python-version }}" - cache-prefix: ${{ inputs.cache-seed }}|relenv|${{ steps.setup-relenv.outputs.version }} - - - build-salt-macos: - name: macOS - if: ${{ inputs.github-hosted-runners }} - strategy: - fail-fast: false - max-parallel: 2 - matrix: - arch: - - x86_64 - runs-on: macos-12 - steps: - - - name: "Throttle Builds" - shell: bash - run: | - t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t" - - - uses: actions/checkout@v4 - - name: Set up Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Setup Relenv - id: setup-relenv - uses: ./.github/actions/setup-relenv - with: - platform: darwin - arch: ${{ matrix.arch }} - version: ${{ inputs.relenv-version }} - cache-seed: ${{ inputs.cache-seed }} - python-version: ${{ inputs.python-version }} - - - name: Setup Python Tools Scripts - uses: ./.github/actions/setup-python-tools-scripts - - - name: Setup Salt Version - id: setup-salt-version - uses: ./.github/actions/setup-salt-version - with: - salt-version: "${{ inputs.salt-version }}" - - - name: Install Salt into Relenv Onedir - uses: ./.github/actions/build-onedir-salt - with: - platform: darwin - arch: ${{ matrix.arch }} - salt-version: "${{ inputs.salt-version }}" - python-version: "${{ inputs.python-version }}" - cache-prefix: ${{ inputs.cache-seed }}|relenv|${{ steps.setup-relenv.outputs.version }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c2fe347233..b5e9b526c17 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -473,16 +473,46 @@ jobs: relenv-version: "0.13.11" python-version: "3.10.13" - build-salt-onedir: + build-salt-onedir-linux: name: Build Salt Onedir if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} needs: - prepare-workflow - build-deps-onedir-linux + - build-source-tarball + uses: ./.github/workflows/build-salt-onedir-linux.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-salt-onedir-windows: + name: Build Salt Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} + needs: + - prepare-workflow - build-deps-onedir-windows + - build-source-tarball + uses: ./.github/workflows/build-salt-onedir-windows.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-salt-onedir-macos: + name: Build Salt Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} + needs: + - prepare-workflow - build-deps-onedir-macos - build-source-tarball - uses: ./.github/workflows/build-salt-onedir.yml + uses: ./.github/workflows/build-salt-onedir-macos.yml with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -496,7 +526,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-rpm-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -509,7 +539,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-rpm-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -522,7 +552,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deb-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -535,7 +565,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deb-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -548,7 +578,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-windows-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -561,7 +591,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-windows-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -574,7 +604,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-macos uses: ./.github/workflows/build-macos-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -587,7 +617,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-macos uses: ./.github/workflows/build-macos-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -600,7 +630,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: windows-2016 @@ -617,7 +647,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: windows-2019 @@ -634,7 +664,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: windows-2022 @@ -651,7 +681,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-macos uses: ./.github/workflows/build-deps-ci-action-macos.yml with: distro-slug: macos-12 @@ -668,7 +698,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-8 @@ -685,7 +715,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-8-arm64 @@ -702,7 +732,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-9 @@ -719,7 +749,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-9-arm64 @@ -736,7 +766,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: amazonlinux-2 @@ -753,7 +783,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: amazonlinux-2-arm64 @@ -770,7 +800,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: archlinux-lts @@ -787,7 +817,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centos-7 @@ -804,7 +834,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centos-7-arm64 @@ -821,7 +851,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-8 @@ -838,7 +868,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-8-arm64 @@ -855,7 +885,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-9 @@ -872,7 +902,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-9-arm64 @@ -889,7 +919,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: debian-10 @@ -906,7 +936,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: debian-11 @@ -923,7 +953,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: debian-11-arm64 @@ -940,7 +970,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-37 @@ -957,7 +987,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-37-arm64 @@ -974,7 +1004,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-38 @@ -991,7 +1021,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-38-arm64 @@ -1008,7 +1038,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: opensuse-15 @@ -1025,7 +1055,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: photonos-3 @@ -1042,7 +1072,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: photonos-4 @@ -1059,7 +1089,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: photonos-4-arm64 @@ -1076,7 +1106,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-20.04 @@ -1093,7 +1123,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-20.04-arm64 @@ -1110,7 +1140,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-22.04 @@ -1127,7 +1157,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-22.04-arm64 @@ -2208,7 +2238,9 @@ jobs: - build-deps-onedir-linux - build-deps-onedir-windows - build-deps-onedir-macos - - build-salt-onedir + - build-salt-onedir-linux + - build-salt-onedir-windows + - build-salt-onedir-macos - build-rpm-pkgs-src - build-deb-pkgs-src - build-windows-pkgs-src diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5c52d2717fe..e094519215d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -522,16 +522,46 @@ jobs: relenv-version: "0.13.11" python-version: "3.10.13" - build-salt-onedir: + build-salt-onedir-linux: name: Build Salt Onedir if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} needs: - prepare-workflow - build-deps-onedir-linux + - build-source-tarball + uses: ./.github/workflows/build-salt-onedir-linux.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-salt-onedir-windows: + name: Build Salt Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} + needs: + - prepare-workflow - build-deps-onedir-windows + - build-source-tarball + uses: ./.github/workflows/build-salt-onedir-windows.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-salt-onedir-macos: + name: Build Salt Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} + needs: + - prepare-workflow - build-deps-onedir-macos - build-source-tarball - uses: ./.github/workflows/build-salt-onedir.yml + uses: ./.github/workflows/build-salt-onedir-macos.yml with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -545,7 +575,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-rpm-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -558,7 +588,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-rpm-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -571,7 +601,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deb-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -584,7 +614,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deb-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -597,7 +627,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-windows-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -613,7 +643,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-windows-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -629,7 +659,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-macos uses: ./.github/workflows/build-macos-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -645,7 +675,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-macos uses: ./.github/workflows/build-macos-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -661,7 +691,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: windows-2016 @@ -678,7 +708,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: windows-2019 @@ -695,7 +725,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: windows-2022 @@ -712,7 +742,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-macos uses: ./.github/workflows/build-deps-ci-action-macos.yml with: distro-slug: macos-12 @@ -729,7 +759,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-8 @@ -746,7 +776,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-8-arm64 @@ -763,7 +793,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-9 @@ -780,7 +810,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-9-arm64 @@ -797,7 +827,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: amazonlinux-2 @@ -814,7 +844,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: amazonlinux-2-arm64 @@ -831,7 +861,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: archlinux-lts @@ -848,7 +878,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centos-7 @@ -865,7 +895,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centos-7-arm64 @@ -882,7 +912,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-8 @@ -899,7 +929,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-8-arm64 @@ -916,7 +946,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-9 @@ -933,7 +963,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-9-arm64 @@ -950,7 +980,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: debian-10 @@ -967,7 +997,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: debian-11 @@ -984,7 +1014,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: debian-11-arm64 @@ -1001,7 +1031,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-37 @@ -1018,7 +1048,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-37-arm64 @@ -1035,7 +1065,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-38 @@ -1052,7 +1082,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-38-arm64 @@ -1069,7 +1099,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: opensuse-15 @@ -1086,7 +1116,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: photonos-3 @@ -1103,7 +1133,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: photonos-4 @@ -1120,7 +1150,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: photonos-4-arm64 @@ -1137,7 +1167,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-20.04 @@ -1154,7 +1184,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-20.04-arm64 @@ -1171,7 +1201,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-22.04 @@ -1188,7 +1218,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-22.04-arm64 @@ -2264,7 +2294,6 @@ jobs: - repo-nightly needs: - prepare-workflow - - build-salt-onedir steps: - uses: actions/checkout@v4 @@ -2781,7 +2810,6 @@ jobs: - repo-nightly needs: - prepare-workflow - - build-salt-onedir steps: - uses: actions/checkout@v4 @@ -3008,7 +3036,9 @@ jobs: - build-deps-onedir-linux - build-deps-onedir-windows - build-deps-onedir-macos - - build-salt-onedir + - build-salt-onedir-linux + - build-salt-onedir-windows + - build-salt-onedir-macos - build-rpm-pkgs-src - build-deb-pkgs-src - build-windows-pkgs-src diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 85f15461dc3..95cbc29b6e9 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -507,16 +507,46 @@ jobs: relenv-version: "0.13.11" python-version: "3.10.13" - build-salt-onedir: + build-salt-onedir-linux: name: Build Salt Onedir if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} needs: - prepare-workflow - build-deps-onedir-linux + - build-source-tarball + uses: ./.github/workflows/build-salt-onedir-linux.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-salt-onedir-windows: + name: Build Salt Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} + needs: + - prepare-workflow - build-deps-onedir-windows + - build-source-tarball + uses: ./.github/workflows/build-salt-onedir-windows.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-salt-onedir-macos: + name: Build Salt Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} + needs: + - prepare-workflow - build-deps-onedir-macos - build-source-tarball - uses: ./.github/workflows/build-salt-onedir.yml + uses: ./.github/workflows/build-salt-onedir-macos.yml with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -530,7 +560,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-rpm-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -543,7 +573,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-rpm-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -556,7 +586,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deb-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -569,7 +599,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deb-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -582,7 +612,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-windows-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -595,7 +625,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-windows-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -608,7 +638,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-macos uses: ./.github/workflows/build-macos-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -621,7 +651,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-macos uses: ./.github/workflows/build-macos-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -634,7 +664,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: windows-2016 @@ -651,7 +681,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: windows-2019 @@ -668,7 +698,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: windows-2022 @@ -685,7 +715,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-macos uses: ./.github/workflows/build-deps-ci-action-macos.yml with: distro-slug: macos-12 @@ -702,7 +732,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-8 @@ -719,7 +749,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-8-arm64 @@ -736,7 +766,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-9 @@ -753,7 +783,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-9-arm64 @@ -770,7 +800,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: amazonlinux-2 @@ -787,7 +817,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: amazonlinux-2-arm64 @@ -804,7 +834,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: archlinux-lts @@ -821,7 +851,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centos-7 @@ -838,7 +868,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centos-7-arm64 @@ -855,7 +885,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-8 @@ -872,7 +902,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-8-arm64 @@ -889,7 +919,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-9 @@ -906,7 +936,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-9-arm64 @@ -923,7 +953,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: debian-10 @@ -940,7 +970,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: debian-11 @@ -957,7 +987,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: debian-11-arm64 @@ -974,7 +1004,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-37 @@ -991,7 +1021,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-37-arm64 @@ -1008,7 +1038,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-38 @@ -1025,7 +1055,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-38-arm64 @@ -1042,7 +1072,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: opensuse-15 @@ -1059,7 +1089,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: photonos-3 @@ -1076,7 +1106,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: photonos-4 @@ -1093,7 +1123,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: photonos-4-arm64 @@ -1110,7 +1140,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-20.04 @@ -1127,7 +1157,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-20.04-arm64 @@ -1144,7 +1174,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-22.04 @@ -1161,7 +1191,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-22.04-arm64 @@ -2244,7 +2274,9 @@ jobs: - build-deps-onedir-linux - build-deps-onedir-windows - build-deps-onedir-macos - - build-salt-onedir + - build-salt-onedir-linux + - build-salt-onedir-windows + - build-salt-onedir-macos - build-rpm-pkgs-src - build-deb-pkgs-src - build-windows-pkgs-src diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 416940e344c..e68541a55f0 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -517,16 +517,46 @@ jobs: relenv-version: "0.13.11" python-version: "3.10.13" - build-salt-onedir: + build-salt-onedir-linux: name: Build Salt Onedir if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} needs: - prepare-workflow - build-deps-onedir-linux + - build-source-tarball + uses: ./.github/workflows/build-salt-onedir-linux.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-salt-onedir-windows: + name: Build Salt Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} + needs: + - prepare-workflow - build-deps-onedir-windows + - build-source-tarball + uses: ./.github/workflows/build-salt-onedir-windows.yml + with: + cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} + salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" + self-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} + github-hosted-runners: ${{ fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} + relenv-version: "0.13.11" + python-version: "3.10.13" + + build-salt-onedir-macos: + name: Build Salt Onedir + if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-salt-onedir'] }} + needs: + - prepare-workflow - build-deps-onedir-macos - build-source-tarball - uses: ./.github/workflows/build-salt-onedir.yml + uses: ./.github/workflows/build-salt-onedir-macos.yml with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -540,7 +570,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-rpm-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -553,7 +583,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-rpm-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -566,7 +596,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deb-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -579,7 +609,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deb-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -592,7 +622,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-windows-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -608,7 +638,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-windows-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -624,7 +654,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-macos uses: ./.github/workflows/build-macos-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -640,7 +670,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-macos uses: ./.github/workflows/build-macos-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -656,7 +686,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: windows-2016 @@ -673,7 +703,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: windows-2019 @@ -690,7 +720,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: windows-2022 @@ -707,7 +737,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-macos uses: ./.github/workflows/build-deps-ci-action-macos.yml with: distro-slug: macos-12 @@ -724,7 +754,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-8 @@ -741,7 +771,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-8-arm64 @@ -758,7 +788,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-9 @@ -775,7 +805,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: almalinux-9-arm64 @@ -792,7 +822,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: amazonlinux-2 @@ -809,7 +839,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: amazonlinux-2-arm64 @@ -826,7 +856,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: archlinux-lts @@ -843,7 +873,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centos-7 @@ -860,7 +890,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centos-7-arm64 @@ -877,7 +907,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-8 @@ -894,7 +924,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-8-arm64 @@ -911,7 +941,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-9 @@ -928,7 +958,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: centosstream-9-arm64 @@ -945,7 +975,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: debian-10 @@ -962,7 +992,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: debian-11 @@ -979,7 +1009,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: debian-11-arm64 @@ -996,7 +1026,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-37 @@ -1013,7 +1043,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-37-arm64 @@ -1030,7 +1060,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-38 @@ -1047,7 +1077,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: fedora-38-arm64 @@ -1064,7 +1094,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: opensuse-15 @@ -1081,7 +1111,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: photonos-3 @@ -1098,7 +1128,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: photonos-4 @@ -1115,7 +1145,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: photonos-4-arm64 @@ -1132,7 +1162,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-20.04 @@ -1149,7 +1179,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-20.04-arm64 @@ -1166,7 +1196,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-22.04 @@ -1183,7 +1213,7 @@ jobs: if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: ubuntu-22.04-arm64 @@ -2146,7 +2176,6 @@ jobs: - repo-staging needs: - prepare-workflow - - build-salt-onedir steps: - uses: actions/checkout@v4 @@ -2665,7 +2694,6 @@ jobs: - repo-staging needs: - prepare-workflow - - build-salt-onedir steps: - uses: actions/checkout@v4 @@ -3788,7 +3816,9 @@ jobs: - build-deps-onedir-linux - build-deps-onedir-windows - build-deps-onedir-macos - - build-salt-onedir + - build-salt-onedir-linux + - build-salt-onedir-windows + - build-salt-onedir-macos - build-rpm-pkgs-src - build-deb-pkgs-src - build-windows-pkgs-src diff --git a/.github/workflows/templates/build-ci-deps.yml.jinja b/.github/workflows/templates/build-ci-deps.yml.jinja index 928128140a5..7faabaee761 100644 --- a/.github/workflows/templates/build-ci-deps.yml.jinja +++ b/.github/workflows/templates/build-ci-deps.yml.jinja @@ -9,7 +9,7 @@ if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-windows uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: <{ slug }> @@ -31,7 +31,7 @@ if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['github-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-macos uses: ./.github/workflows/build-deps-ci-action-macos.yml with: distro-slug: <{ slug }> @@ -80,7 +80,7 @@ if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['test'] && fromJSON(needs.prepare-workflow.outputs.runners)['self-hosted'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-linux uses: ./.github/workflows/build-deps-ci-action.yml with: distro-slug: <{ slug }> diff --git a/.github/workflows/templates/build-packages.yml.jinja b/.github/workflows/templates/build-packages.yml.jinja index 929bac1109b..37cae00b1e8 100644 --- a/.github/workflows/templates/build-packages.yml.jinja +++ b/.github/workflows/templates/build-packages.yml.jinja @@ -1,9 +1,9 @@ -<%- for pkg_type, display_name, runner_type in ( - ("rpm", "RPM", "self-hosted"), - ("deb", "DEB", "self-hosted"), - ("windows", "Windows", "github-hosted"), - ("macos", "macOS", "github-hosted"), - ) %> +<%- for platform, pkg_type, display_name, runner_type in ( + ("linux", "rpm", "RPM", "self-hosted"), + ("linux", "deb", "DEB", "self-hosted"), + ("windows", "windows", "Windows", "github-hosted"), + ("macos", "macos", "macOS", "github-hosted"), + ) %> <%- for backend in ("onedir", "src") %> <%- set job_name = "build-{}-pkgs-{}".format(pkg_type, backend) %> @@ -16,7 +16,7 @@ if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['build-pkgs'] && fromJSON(needs.prepare-workflow.outputs.runners)['<{ runner_type }>'] }} needs: - prepare-workflow - - build-salt-onedir + - build-salt-onedir-<{ platform }> uses: ./.github/workflows/build-<{ pkg_type }>-packages.yml with: salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" diff --git a/.github/workflows/templates/build-repos.yml.jinja b/.github/workflows/templates/build-repos.yml.jinja index de984ed75de..8aa147ce40b 100644 --- a/.github/workflows/templates/build-repos.yml.jinja +++ b/.github/workflows/templates/build-repos.yml.jinja @@ -1,11 +1,11 @@ -<%- for type, display_name, needs_pkg in ( - ("src", "Source", False), - ("deb", "DEB", True), - ("rpm", "RPM", True), - ("windows", "Windows", True), - ("macos", "macOS", True), - ("onedir", "Onedir", False), - ) %> +<%- for platform, type, display_name, needs_pkg in ( + (None, "src", "Source", False), + ("linux", "deb", "DEB", True), + ("linux", "rpm", "RPM", True), + ("windows", "windows", "Windows", True), + ("macos", "macos", "macOS", True), + (None, "onedir", "Onedir", False), + ) %> <%- set job_name = "build-{}-repo".format(type) %> <%- do build_repo_needs.append(job_name) %> @@ -21,8 +21,8 @@ - prepare-workflow <%- if needs_pkg %> - build-<{ type }>-pkgs-onedir - <%- else %> - - build-salt-onedir + <%- elif platform %> + - build-salt-onedir-<{ platform }> <%- endif %> <%- include "build-{}-repo.yml.jinja".format(type) %> diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index 896bc880354..f09b92f64de 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -278,18 +278,18 @@ <%- set job_name = "build-salt-onedir" %> <%- if includes.get(job_name, True) %> + <%- for platform in ("linux", "windows", "macos") %> + <%- set platform_job_name = "{}-{}".format(job_name, platform) %> - <{ job_name }>: - <%- do conclusion_needs.append(job_name) %> + <{ platform_job_name }>: + <%- do conclusion_needs.append(platform_job_name) %> name: Build Salt Onedir if: ${{ fromJSON(needs.prepare-workflow.outputs.jobs)['<{ job_name }>'] }} needs: - prepare-workflow - <%- for platform in ("linux", "windows", "macos") %> - build-deps-onedir-<{ platform }> - <%- endfor %> - build-source-tarball - uses: ./.github/workflows/build-salt-onedir.yml + uses: ./.github/workflows/build-salt-onedir-<{ platform }>.yml with: cache-seed: ${{ needs.prepare-workflow.outputs.cache-seed }} salt-version: "${{ needs.prepare-workflow.outputs.salt-version }}" @@ -298,6 +298,7 @@ relenv-version: "<{ relenv_version }>" python-version: "<{ python_version }>" + <%- endfor %> <%- endif %> From 44d47b4d8aa1fd810f4593b69e7781452428bc06 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 3 Oct 2023 13:32:58 +0100 Subject: [PATCH 17/31] Remove unused variables Signed-off-by: Pedro Algarvio --- tests/pytests/unit/fileclient/test_fileclient.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/pytests/unit/fileclient/test_fileclient.py b/tests/pytests/unit/fileclient/test_fileclient.py index 20eebb2bd51..f52e74094ea 100644 --- a/tests/pytests/unit/fileclient/test_fileclient.py +++ b/tests/pytests/unit/fileclient/test_fileclient.py @@ -81,14 +81,11 @@ def test_fileclient_context_manager_closes(minion_opts, master_opts): "auth_timeout": 5, "master_ip": "127.0.0.1", "master_port": master_opts["ret_port"], - "master_uri": "tcp://127.0.0.1:{}".format(master_opts["ret_port"]), + "master_uri": f"tcp://127.0.0.1:{master_opts['ret_port']}", "request_channel_timeout": 1, "request_channel_tries": 1, } ) - master_uri = "tcp://{master_ip}:{master_port}".format( - master_ip="localhost", master_port=minion_opts["master_port"] - ) mock_reqchannel = MockReqChannel() patch_reqchannel = patch.object( salt.channel.client, "ReqChannel", return_value=mock_reqchannel @@ -116,14 +113,11 @@ def test_fileclient_timeout(minion_opts, master_opts): "auth_timeout": 5, "master_ip": "127.0.0.1", "master_port": master_opts["ret_port"], - "master_uri": "tcp://127.0.0.1:{}".format(master_opts["ret_port"]), + "master_uri": f"tcp://127.0.0.1:{master_opts['ret_port']}", "request_channel_timeout": 1, "request_channel_tries": 1, } ) - master_uri = "tcp://{master_ip}:{master_port}".format( - master_ip="localhost", master_port=minion_opts["master_port"] - ) async def mock_auth(): return True From 37ca0c08d0f7dfd83477f0456ddc1dd400beb6dd Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 3 Oct 2023 13:55:03 +0100 Subject: [PATCH 18/31] Upgrade to `mock==5.1.0` Signed-off-by: Pedro Algarvio --- requirements/static/ci/py3.10/cloud.txt | 3 +-- requirements/static/ci/py3.10/darwin.txt | 3 +-- requirements/static/ci/py3.10/freebsd.txt | 3 +-- requirements/static/ci/py3.10/linux.txt | 3 +-- requirements/static/ci/py3.10/windows.txt | 3 +-- requirements/static/ci/py3.7/cloud.txt | 3 +-- requirements/static/ci/py3.7/freebsd.txt | 3 +-- requirements/static/ci/py3.7/linux.txt | 3 +-- requirements/static/ci/py3.7/windows.txt | 3 +-- requirements/static/ci/py3.8/cloud.txt | 3 +-- requirements/static/ci/py3.8/freebsd.txt | 3 +-- requirements/static/ci/py3.8/linux.txt | 3 +-- requirements/static/ci/py3.8/windows.txt | 3 +-- requirements/static/ci/py3.9/cloud.txt | 3 +-- requirements/static/ci/py3.9/darwin.txt | 3 +-- requirements/static/ci/py3.9/freebsd.txt | 3 +-- requirements/static/ci/py3.9/linux.txt | 3 +-- requirements/static/ci/py3.9/windows.txt | 3 +-- tests/support/mock.py | 1 + 19 files changed, 19 insertions(+), 36 deletions(-) diff --git a/requirements/static/ci/py3.10/cloud.txt b/requirements/static/ci/py3.10/cloud.txt index 8191c938ac1..1179cd38b7d 100644 --- a/requirements/static/ci/py3.10/cloud.txt +++ b/requirements/static/ci/py3.10/cloud.txt @@ -788,7 +788,7 @@ markupsafe==2.1.2 # mako # moto # werkzeug -mock==3.0.5 +mock==5.1.0 # via # -c requirements/static/ci/py3.10/linux.txt # -r requirements/pytest.txt @@ -1212,7 +1212,6 @@ six==1.16.0 # junos-eznc # kazoo # kubernetes - # mock # more-itertools # ncclient # paramiko diff --git a/requirements/static/ci/py3.10/darwin.txt b/requirements/static/ci/py3.10/darwin.txt index 47b122502d4..256fe689592 100644 --- a/requirements/static/ci/py3.10/darwin.txt +++ b/requirements/static/ci/py3.10/darwin.txt @@ -536,7 +536,7 @@ markupsafe==2.1.2 # werkzeug mercurial==6.0.1 # via -r requirements/static/ci/darwin.in -mock==3.0.5 +mock==5.1.0 # via -r requirements/pytest.txt more-itertools==8.2.0 # via @@ -855,7 +855,6 @@ six==1.16.0 # jsonschema # junos-eznc # kubernetes - # mock # msrestazure # ncclient # paramiko diff --git a/requirements/static/ci/py3.10/freebsd.txt b/requirements/static/ci/py3.10/freebsd.txt index 693b0fa2bc0..18f4f42651c 100644 --- a/requirements/static/ci/py3.10/freebsd.txt +++ b/requirements/static/ci/py3.10/freebsd.txt @@ -528,7 +528,7 @@ markupsafe==2.1.2 # werkzeug mercurial==6.0.1 # via -r requirements/static/ci/freebsd.in -mock==3.0.5 +mock==5.1.0 # via -r requirements/pytest.txt more-itertools==5.0.0 # via @@ -848,7 +848,6 @@ six==1.16.0 # junos-eznc # kazoo # kubernetes - # mock # more-itertools # msrestazure # ncclient diff --git a/requirements/static/ci/py3.10/linux.txt b/requirements/static/ci/py3.10/linux.txt index 2caf2e6fd3a..0049950e1ef 100644 --- a/requirements/static/ci/py3.10/linux.txt +++ b/requirements/static/ci/py3.10/linux.txt @@ -543,7 +543,7 @@ markupsafe==2.1.2 # werkzeug mercurial==6.0.1 # via -r requirements/static/ci/linux.in -mock==3.0.5 +mock==5.1.0 # via -r requirements/pytest.txt more-itertools==5.0.0 # via @@ -889,7 +889,6 @@ six==1.16.0 # junos-eznc # kazoo # kubernetes - # mock # more-itertools # ncclient # paramiko diff --git a/requirements/static/ci/py3.10/windows.txt b/requirements/static/ci/py3.10/windows.txt index d7c65b7e59c..92a4b88303b 100644 --- a/requirements/static/ci/py3.10/windows.txt +++ b/requirements/static/ci/py3.10/windows.txt @@ -208,7 +208,7 @@ markupsafe==2.1.2 # mako # moto # werkzeug -mock==3.0.5 +mock==5.1.0 # via -r requirements/pytest.txt more-itertools==8.2.0 # via @@ -420,7 +420,6 @@ six==1.15.0 # google-auth # jsonschema # kubernetes - # mock # python-dateutil # pyvmomi # pywinrm diff --git a/requirements/static/ci/py3.7/cloud.txt b/requirements/static/ci/py3.7/cloud.txt index 3f64dfca7d5..97736d31111 100644 --- a/requirements/static/ci/py3.7/cloud.txt +++ b/requirements/static/ci/py3.7/cloud.txt @@ -818,7 +818,7 @@ markupsafe==2.1.2 # mako # moto # werkzeug -mock==3.0.5 +mock==5.1.0 # via # -c requirements/static/ci/py3.7/linux.txt # -r requirements/pytest.txt @@ -1266,7 +1266,6 @@ six==1.16.0 # junos-eznc # kazoo # kubernetes - # mock # more-itertools # ncclient # paramiko diff --git a/requirements/static/ci/py3.7/freebsd.txt b/requirements/static/ci/py3.7/freebsd.txt index 9d27b43d5b2..a979abdfb10 100644 --- a/requirements/static/ci/py3.7/freebsd.txt +++ b/requirements/static/ci/py3.7/freebsd.txt @@ -552,7 +552,7 @@ markupsafe==2.1.2 # werkzeug mercurial==6.0.1 # via -r requirements/static/ci/freebsd.in -mock==3.0.5 +mock==5.1.0 # via -r requirements/pytest.txt more-itertools==5.0.0 # via @@ -892,7 +892,6 @@ six==1.16.0 # junos-eznc # kazoo # kubernetes - # mock # more-itertools # msrestazure # ncclient diff --git a/requirements/static/ci/py3.7/linux.txt b/requirements/static/ci/py3.7/linux.txt index c9f7362187e..6f648598b31 100644 --- a/requirements/static/ci/py3.7/linux.txt +++ b/requirements/static/ci/py3.7/linux.txt @@ -567,7 +567,7 @@ markupsafe==2.1.2 # werkzeug mercurial==6.0.1 # via -r requirements/static/ci/linux.in -mock==3.0.5 +mock==5.1.0 # via -r requirements/pytest.txt more-itertools==5.0.0 # via @@ -933,7 +933,6 @@ six==1.16.0 # junos-eznc # kazoo # kubernetes - # mock # more-itertools # ncclient # paramiko diff --git a/requirements/static/ci/py3.7/windows.txt b/requirements/static/ci/py3.7/windows.txt index b240ba476ad..3a785589f0d 100644 --- a/requirements/static/ci/py3.7/windows.txt +++ b/requirements/static/ci/py3.7/windows.txt @@ -222,7 +222,7 @@ markupsafe==2.1.2 # mako # moto # werkzeug -mock==3.0.5 +mock==5.1.0 # via -r requirements/pytest.txt more-itertools==8.2.0 # via @@ -434,7 +434,6 @@ six==1.15.0 # google-auth # jsonschema # kubernetes - # mock # python-dateutil # pyvmomi # pywinrm diff --git a/requirements/static/ci/py3.8/cloud.txt b/requirements/static/ci/py3.8/cloud.txt index 0d2631e1fcc..5c0e53d37a7 100644 --- a/requirements/static/ci/py3.8/cloud.txt +++ b/requirements/static/ci/py3.8/cloud.txt @@ -806,7 +806,7 @@ markupsafe==2.1.2 # mako # moto # werkzeug -mock==3.0.5 +mock==5.1.0 # via # -c requirements/static/ci/py3.8/linux.txt # -r requirements/pytest.txt @@ -1254,7 +1254,6 @@ six==1.16.0 # junos-eznc # kazoo # kubernetes - # mock # more-itertools # ncclient # paramiko diff --git a/requirements/static/ci/py3.8/freebsd.txt b/requirements/static/ci/py3.8/freebsd.txt index c6d484ac22f..0a035c08cdf 100644 --- a/requirements/static/ci/py3.8/freebsd.txt +++ b/requirements/static/ci/py3.8/freebsd.txt @@ -542,7 +542,7 @@ markupsafe==2.1.2 # werkzeug mercurial==6.0.1 # via -r requirements/static/ci/freebsd.in -mock==3.0.5 +mock==5.1.0 # via -r requirements/pytest.txt more-itertools==5.0.0 # via @@ -882,7 +882,6 @@ six==1.16.0 # junos-eznc # kazoo # kubernetes - # mock # more-itertools # msrestazure # ncclient diff --git a/requirements/static/ci/py3.8/linux.txt b/requirements/static/ci/py3.8/linux.txt index bc1caeeba8c..337f491ddc5 100644 --- a/requirements/static/ci/py3.8/linux.txt +++ b/requirements/static/ci/py3.8/linux.txt @@ -557,7 +557,7 @@ markupsafe==2.1.2 # werkzeug mercurial==6.0.1 # via -r requirements/static/ci/linux.in -mock==3.0.5 +mock==5.1.0 # via -r requirements/pytest.txt more-itertools==5.0.0 # via @@ -923,7 +923,6 @@ six==1.16.0 # junos-eznc # kazoo # kubernetes - # mock # more-itertools # ncclient # paramiko diff --git a/requirements/static/ci/py3.8/windows.txt b/requirements/static/ci/py3.8/windows.txt index ff143d4fcff..244a1e0734f 100644 --- a/requirements/static/ci/py3.8/windows.txt +++ b/requirements/static/ci/py3.8/windows.txt @@ -210,7 +210,7 @@ markupsafe==2.1.2 # mako # moto # werkzeug -mock==3.0.5 +mock==5.1.0 # via -r requirements/pytest.txt more-itertools==8.2.0 # via @@ -422,7 +422,6 @@ six==1.15.0 # google-auth # jsonschema # kubernetes - # mock # python-dateutil # pyvmomi # pywinrm diff --git a/requirements/static/ci/py3.9/cloud.txt b/requirements/static/ci/py3.9/cloud.txt index 97ed2d28172..47593bdb4f1 100644 --- a/requirements/static/ci/py3.9/cloud.txt +++ b/requirements/static/ci/py3.9/cloud.txt @@ -806,7 +806,7 @@ markupsafe==2.1.2 # mako # moto # werkzeug -mock==3.0.5 +mock==5.1.0 # via # -c requirements/static/ci/py3.9/linux.txt # -r requirements/pytest.txt @@ -1256,7 +1256,6 @@ six==1.16.0 # junos-eznc # kazoo # kubernetes - # mock # more-itertools # ncclient # paramiko diff --git a/requirements/static/ci/py3.9/darwin.txt b/requirements/static/ci/py3.9/darwin.txt index bd6610b93ef..05970bc3186 100644 --- a/requirements/static/ci/py3.9/darwin.txt +++ b/requirements/static/ci/py3.9/darwin.txt @@ -550,7 +550,7 @@ markupsafe==2.1.2 # werkzeug mercurial==6.0.1 # via -r requirements/static/ci/darwin.in -mock==3.0.5 +mock==5.1.0 # via -r requirements/pytest.txt more-itertools==8.2.0 # via @@ -891,7 +891,6 @@ six==1.16.0 # jsonschema # junos-eznc # kubernetes - # mock # msrestazure # ncclient # paramiko diff --git a/requirements/static/ci/py3.9/freebsd.txt b/requirements/static/ci/py3.9/freebsd.txt index 5b2e19b1359..1e75e0a42d8 100644 --- a/requirements/static/ci/py3.9/freebsd.txt +++ b/requirements/static/ci/py3.9/freebsd.txt @@ -542,7 +542,7 @@ markupsafe==2.1.2 # werkzeug mercurial==6.0.1 # via -r requirements/static/ci/freebsd.in -mock==3.0.5 +mock==5.1.0 # via -r requirements/pytest.txt more-itertools==5.0.0 # via @@ -884,7 +884,6 @@ six==1.16.0 # junos-eznc # kazoo # kubernetes - # mock # more-itertools # msrestazure # ncclient diff --git a/requirements/static/ci/py3.9/linux.txt b/requirements/static/ci/py3.9/linux.txt index f8c2acc2047..8d2c2f6d54d 100644 --- a/requirements/static/ci/py3.9/linux.txt +++ b/requirements/static/ci/py3.9/linux.txt @@ -559,7 +559,7 @@ markupsafe==2.1.2 # werkzeug mercurial==6.0.1 # via -r requirements/static/ci/linux.in -mock==3.0.5 +mock==5.1.0 # via -r requirements/pytest.txt more-itertools==5.0.0 # via @@ -927,7 +927,6 @@ six==1.16.0 # junos-eznc # kazoo # kubernetes - # mock # more-itertools # ncclient # paramiko diff --git a/requirements/static/ci/py3.9/windows.txt b/requirements/static/ci/py3.9/windows.txt index d749dd23ae4..74979a411af 100644 --- a/requirements/static/ci/py3.9/windows.txt +++ b/requirements/static/ci/py3.9/windows.txt @@ -210,7 +210,7 @@ markupsafe==2.1.2 # mako # moto # werkzeug -mock==3.0.5 +mock==5.1.0 # via -r requirements/pytest.txt more-itertools==8.2.0 # via @@ -423,7 +423,6 @@ six==1.15.0 # google-auth # jsonschema # kubernetes - # mock # python-dateutil # pyvmomi # pywinrm diff --git a/tests/support/mock.py b/tests/support/mock.py index 2256ad8f5da..c050d0bf4e6 100644 --- a/tests/support/mock.py +++ b/tests/support/mock.py @@ -27,6 +27,7 @@ from mock import ( ANY, DEFAULT, FILTER_DIR, + AsyncMock, MagicMock, Mock, NonCallableMagicMock, From 669ce9b49c6d9aac8ae3fffe0d3efa2c584a094c Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 3 Oct 2023 13:57:07 +0100 Subject: [PATCH 19/31] Properly mock async method Signed-off-by: Pedro Algarvio --- .../unit/fileclient/test_fileclient.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/tests/pytests/unit/fileclient/test_fileclient.py b/tests/pytests/unit/fileclient/test_fileclient.py index f52e74094ea..0e072e5c36f 100644 --- a/tests/pytests/unit/fileclient/test_fileclient.py +++ b/tests/pytests/unit/fileclient/test_fileclient.py @@ -1,8 +1,6 @@ """ Tests for the salt fileclient """ - - import errno import logging import os @@ -11,7 +9,7 @@ import pytest import salt.utils.files from salt import fileclient -from tests.support.mock import MagicMock, Mock, patch +from tests.support.mock import AsyncMock, MagicMock, Mock, patch log = logging.getLogger(__name__) @@ -119,20 +117,17 @@ def test_fileclient_timeout(minion_opts, master_opts): } ) - async def mock_auth(): - return True - def mock_dumps(*args): return b"meh" with fileclient.get_file_client(minion_opts) as client: # Authenticate must return true - client.auth.authenticate = mock_auth - # Crypticle must return bytes to pass to transport.RequestClient.send - client.auth._crypticle = Mock() - client.auth._crypticle.dumps = mock_dumps - with pytest.raises(salt.exceptions.SaltClientError): - client.file_list() + with patch.object(client.auth, "authenticate", AsyncMock(return_value=True)): + # Crypticle must return bytes to pass to transport.RequestClient.send + client.auth._crypticle = Mock() + client.auth._crypticle.dumps = mock_dumps + with pytest.raises(salt.exceptions.SaltClientError): + client.file_list() def test_cache_skips_makedirs_on_race_condition(client_opts): From 3a4df003d95a3e5e3c1a688c1f20ecaf1ad41f6e Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 13 Jun 2023 07:37:19 +0100 Subject: [PATCH 20/31] Fix mock calls Signed-off-by: Pedro Algarvio --- tests/pytests/unit/modules/test_aptpkg.py | 12 +++++----- .../pytests/unit/modules/test_linux_sysctl.py | 8 +++---- tests/pytests/unit/modules/test_win_ip.py | 4 ++-- tests/pytests/unit/test_master.py | 2 +- tests/pytests/unit/utils/event/test_event.py | 24 +++++++++---------- tests/unit/modules/test_nilrt_ip.py | 4 ++-- .../unit/netapi/rest_tornado/test_saltnado.py | 22 +++++++---------- 7 files changed, 35 insertions(+), 41 deletions(-) diff --git a/tests/pytests/unit/modules/test_aptpkg.py b/tests/pytests/unit/modules/test_aptpkg.py index 33d976b8427..9a812d817ce 100644 --- a/tests/pytests/unit/modules/test_aptpkg.py +++ b/tests/pytests/unit/modules/test_aptpkg.py @@ -1298,17 +1298,17 @@ def test_call_apt_dpkg_lock(): ] cmd_mock = MagicMock(side_effect=cmd_side_effect) - cmd_call = ( + cmd_call = [ call( ["dpkg", "-l", "python"], - env={}, - ignore_retcode=False, output_loglevel="quiet", python_shell=True, + env={}, + ignore_retcode=False, username="Darth Vader", ), - ) - expected_calls = [cmd_call * 5] + ] + expected_calls = cmd_call * 5 with patch.dict( aptpkg.__salt__, @@ -1328,7 +1328,7 @@ def test_call_apt_dpkg_lock(): # We should attempt to call the cmd 5 times assert cmd_mock.call_count == 5 - cmd_mock.has_calls(expected_calls) + cmd_mock.assert_has_calls(expected_calls) def test_services_need_restart_checkrestart_missing(): diff --git a/tests/pytests/unit/modules/test_linux_sysctl.py b/tests/pytests/unit/modules/test_linux_sysctl.py index 0bdd24039d7..6b0875bc460 100644 --- a/tests/pytests/unit/modules/test_linux_sysctl.py +++ b/tests/pytests/unit/modules/test_linux_sysctl.py @@ -215,7 +215,7 @@ def test_persist_no_conf_failure(): ): with pytest.raises(CommandExecutionError): linux_sysctl.persist("net.ipv4.ip_forward", 42, config=None) - fopen_mock.called_once() + fopen_mock.assert_called_once() def test_persist_no_conf_success(): @@ -353,7 +353,7 @@ def test_persist_value_with_spaces_already_set(tmp_path): """ config = str(tmp_path / "existing_sysctl_with_spaces.conf") value = "|/usr/share/kdump-tools/dump-core %p %s %t %e" - config_file_content = "kernel.core_pattern = {}\n".format(value) + config_file_content = f"kernel.core_pattern = {value}\n" with fopen(config, "w", encoding="utf-8") as config_file: config_file.write(config_file_content) mock_run = MagicMock(return_value=value) @@ -383,7 +383,7 @@ def test_persist_value_with_spaces_already_configured(tmp_path): """ config = str(tmp_path / "existing_sysctl_with_spaces.conf") value = "|/usr/share/kdump-tools/dump-core %p %s %t %e" - config_file_content = "kernel.core_pattern = {}\n".format(value) + config_file_content = f"kernel.core_pattern = {value}\n" with fopen(config, "w", encoding="utf-8") as config_file: config_file.write(config_file_content) mock_run = MagicMock(return_value="") @@ -451,7 +451,7 @@ def test_persist_value_with_spaces_update_config(tmp_path): assert os.path.isfile(config) with fopen(config, encoding="utf-8") as config_file: written = config_file.read() - assert written == "kernel.core_pattern = {}\n".format(value) + assert written == f"kernel.core_pattern = {value}\n" def test_persist_value_with_spaces_new_file(tmp_path): diff --git a/tests/pytests/unit/modules/test_win_ip.py b/tests/pytests/unit/modules/test_win_ip.py index 38eb6b1ac5f..94a3fe7ca93 100644 --- a/tests/pytests/unit/modules/test_win_ip.py +++ b/tests/pytests/unit/modules/test_win_ip.py @@ -151,7 +151,7 @@ def test_enable(): ): assert win_ip.enable("Ethernet") - mock_cmd.called_once_with( + mock_cmd.assert_called_once_with( [ "netsh", "interface", @@ -180,7 +180,7 @@ def test_disable(): ): assert win_ip.disable("Ethernet") - mock_cmd.called_once_with( + mock_cmd.assert_called_once_with( [ "netsh", "interface", diff --git a/tests/pytests/unit/test_master.py b/tests/pytests/unit/test_master.py index cd11d217c79..502767d3e34 100644 --- a/tests/pytests/unit/test_master.py +++ b/tests/pytests/unit/test_master.py @@ -60,7 +60,7 @@ def test_fileserver_duration(): end = time.time() # Interval is equal to timeout so the _do_update method will be called # one time. - update.called_once() + update.assert_called_once() # Timeout is 1 second duration = end - start if duration > 2 and salt.utils.platform.spawning_platform(): diff --git a/tests/pytests/unit/utils/event/test_event.py b/tests/pytests/unit/utils/event/test_event.py index e289e72dad0..f4b6c159996 100644 --- a/tests/pytests/unit/utils/event/test_event.py +++ b/tests/pytests/unit/utils/event/test_event.py @@ -38,7 +38,7 @@ def sock_dir(tmp_path): def _assert_got_event(evt, data, msg=None, expected_failure=False): assert evt is not None, msg for key in data: - assert key in evt, "{}: Key {} missing".format(msg, key) + assert key in evt, f"{msg}: Key {key} missing" assertMsg = "{0}: Key {1} value mismatch, {2} != {3}" assertMsg = assertMsg.format(msg, key, data[key], evt[key]) if not expected_failure: @@ -59,8 +59,8 @@ def test_minion_event(sock_dir): :10 ] with salt.utils.event.MinionEvent(opts, listen=False) as me: - assert me.puburi == str(sock_dir / "minion_event_{}_pub.ipc".format(id_hash)) - assert me.pulluri == str(sock_dir / "minion_event_{}_pull.ipc".format(id_hash)) + assert me.puburi == str(sock_dir / f"minion_event_{id_hash}_pub.ipc") + assert me.pulluri == str(sock_dir / f"minion_event_{id_hash}_pull.ipc") def test_minion_event_tcp_ipc_mode(): @@ -73,8 +73,8 @@ def test_minion_event_tcp_ipc_mode(): def test_minion_event_no_id(sock_dir): with salt.utils.event.MinionEvent(dict(sock_dir=str(sock_dir)), listen=False) as me: id_hash = hashlib.sha256(salt.utils.stringutils.to_bytes("")).hexdigest()[:10] - assert me.puburi == str(sock_dir / "minion_event_{}_pub.ipc".format(id_hash)) - assert me.pulluri == str(sock_dir / "minion_event_{}_pull.ipc".format(id_hash)) + assert me.puburi == str(sock_dir / f"minion_event_{id_hash}_pub.ipc") + assert me.pulluri == str(sock_dir / f"minion_event_{id_hash}_pull.ipc") @pytest.mark.slow_test @@ -256,9 +256,9 @@ def test_event_many(sock_dir): with eventpublisher_process(str(sock_dir)): with salt.utils.event.MasterEvent(str(sock_dir), listen=True) as me: for i in range(500): - me.fire_event({"data": "{}".format(i)}, "testevents") + me.fire_event({"data": f"{i}"}, "testevents") evt = me.get_event(tag="testevents") - _assert_got_event(evt, {"data": "{}".format(i)}, "Event {}".format(i)) + _assert_got_event(evt, {"data": f"{i}"}, f"Event {i}") @pytest.mark.slow_test @@ -268,10 +268,10 @@ def test_event_many_backlog(sock_dir): with salt.utils.event.MasterEvent(str(sock_dir), listen=True) as me: # Must not exceed zmq HWM for i in range(500): - me.fire_event({"data": "{}".format(i)}, "testevents") + me.fire_event({"data": f"{i}"}, "testevents") for i in range(500): evt = me.get_event(tag="testevents") - _assert_got_event(evt, {"data": "{}".format(i)}, "Event {}".format(i)) + _assert_got_event(evt, {"data": f"{i}"}, f"Event {i}") # Test the fire_master function. As it wraps the underlying fire_event, @@ -300,7 +300,7 @@ def test_connect_pull_should_debug_log_on_StreamClosedError(): event = SaltEvent(node=None) with patch.object(event, "pusher") as mock_pusher: with patch.object( - salt.utils.event.log, "debug", auto_spec=True + salt.utils.event.log, "debug", autospec=True ) as mock_log_debug: mock_pusher.connect.side_effect = ( salt.ext.tornado.iostream.StreamClosedError @@ -317,10 +317,10 @@ def test_connect_pull_should_error_log_on_other_errors(error): event = SaltEvent(node=None) with patch.object(event, "pusher") as mock_pusher: with patch.object( - salt.utils.event.log, "debug", auto_spec=True + salt.utils.event.log, "debug", autospec=True ) as mock_log_debug: with patch.object( - salt.utils.event.log, "error", auto_spec=True + salt.utils.event.log, "error", autospec=True ) as mock_log_error: mock_pusher.connect.side_effect = error event.connect_pull() diff --git a/tests/unit/modules/test_nilrt_ip.py b/tests/unit/modules/test_nilrt_ip.py index 1261473edb4..50dc13b20b8 100644 --- a/tests/unit/modules/test_nilrt_ip.py +++ b/tests/unit/modules/test_nilrt_ip.py @@ -28,7 +28,7 @@ class NilrtIPTestCase(TestCase, LoaderModuleMockMixin): "salt.modules.nilrt_ip._change_dhcp_config", return_value=True ) as change_dhcp_config_mock: assert nilrt_ip._change_state("test_interface", "down") - assert change_dhcp_config_mock.called_with("test_interface", False) + change_dhcp_config_mock.assert_called_with("test_interface", False) def test_change_state_up_state(self): """ @@ -42,7 +42,7 @@ class NilrtIPTestCase(TestCase, LoaderModuleMockMixin): "salt.modules.nilrt_ip._change_dhcp_config", return_value=True ) as change_dhcp_config_mock: assert nilrt_ip._change_state("test_interface", "up") - assert change_dhcp_config_mock.called_with("test_interface") + change_dhcp_config_mock.assert_called_with("test_interface") def test_set_static_all_with_dns(self): """ diff --git a/tests/unit/netapi/rest_tornado/test_saltnado.py b/tests/unit/netapi/rest_tornado/test_saltnado.py index 7b63a65d4f3..c4758e700ab 100644 --- a/tests/unit/netapi/rest_tornado/test_saltnado.py +++ b/tests/unit/netapi/rest_tornado/test_saltnado.py @@ -647,7 +647,6 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): with patch.object( self.handler.application.event_listener, "get_event", - autospec=True, side_effect=fancy_get_event, ), patch.dict( self.handler.application.opts, @@ -698,7 +697,6 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): with patch.object( self.handler.application.event_listener, "get_event", - autospec=True, side_effect=fancy_get_event, ), patch.object( self.handler, @@ -729,8 +727,8 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): { "tag": "fnord", "data": { - "return": "return from fnord {}".format(i), - "id": "fnord {}".format(i), + "return": f"return from fnord {i}", + "id": f"fnord {i}", }, } ) @@ -760,7 +758,6 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): with patch.object( self.handler.application.event_listener, "get_event", - autospec=True, side_effect=fancy_get_event, ), patch.object( self.handler, @@ -794,8 +791,8 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): { "tag": "fnord", "data": { - "return": "return from fnord {}".format(i), - "id": "fnord {}".format(i), + "return": f"return from fnord {i}", + "id": f"fnord {i}", }, } ) @@ -820,7 +817,6 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): with patch.object( self.handler.application.event_listener, "get_event", - autospec=True, side_effect=fancy_get_event, ), patch.dict( self.handler.application.opts, @@ -843,12 +839,12 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): completed_events = [salt.ext.tornado.gen.Future() for _ in range(10)] events_by_id = {} for i, event in enumerate(completed_events): - id_ = "fnord {}".format(i) + id_ = f"fnord {i}" events_by_id[id_] = event event.set_result( { "tag": "fnord", - "data": {"return": "return from {}".format(id_), "id": id_}, + "data": {"return": f"return from {id_}", "id": id_}, } ) expected_result = { @@ -878,7 +874,6 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): with patch.object( self.handler.application.event_listener, "get_event", - autospec=True, side_effect=fancy_get_event, ), patch.dict( self.handler.application.opts, @@ -904,12 +899,12 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): events_by_id = {} # Setup some real-enough looking return data for i, event in enumerate(completed_events): - id_ = "fnord {}".format(i) + id_ = f"fnord {i}" events_by_id[id_] = event event.set_result( { "tag": "fnord", - "data": {"return": "return from {}".format(id_), "id": id_}, + "data": {"return": f"return from {id_}", "id": id_}, } ) # Hard coded instead of dynamic to avoid potentially writing a test @@ -971,7 +966,6 @@ class TestDisbatchLocal(salt.ext.tornado.testing.AsyncTestCase): with patch.object( self.handler.application.event_listener, "get_event", - autospec=True, side_effect=fancy_get_event, ), patch.object( self.handler, From 9f2d4cfe3cf965f3e872ecd4cc492537c1e006c3 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 12 Sep 2023 13:04:03 +0100 Subject: [PATCH 21/31] The correct method is `assert_called_once_with` not `called_once_with` Signed-off-by: Pedro Algarvio --- tests/pytests/unit/modules/test_msteams.py | 2 +- tests/pytests/unit/states/test_file.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytests/unit/modules/test_msteams.py b/tests/pytests/unit/modules/test_msteams.py index 05e6a3d2284..1307cd854bd 100644 --- a/tests/pytests/unit/modules/test_msteams.py +++ b/tests/pytests/unit/modules/test_msteams.py @@ -24,7 +24,7 @@ def test_post_card(): with patch("salt.utils.http.query", http_mock): ret = msteams.post_card("test") assert ret - assert http_mock.called_once_with( + assert http_mock.assert_called_once_with( method="POST", header_dict={"Content-Type": "application/json"}, data='{"text": "test", "title": Null, "themeColor": Null}', diff --git a/tests/pytests/unit/states/test_file.py b/tests/pytests/unit/states/test_file.py index bb040f0fc60..393ede9b397 100644 --- a/tests/pytests/unit/states/test_file.py +++ b/tests/pytests/unit/states/test_file.py @@ -87,6 +87,6 @@ def test_file_recurse_directory_test(): "name": "/tmp/test", "result": False, } - assert salt_dunder["cp.list_master_dirs"].called_once_with( + assert salt_dunder["cp.list_master_dirs"].assert_called_once_with( prefix="does_not_exist/" ) From 6c762d7fb158ab76a17f263ef0b52371d446afb6 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 14 Sep 2023 08:03:14 +0100 Subject: [PATCH 22/31] Fix failing tests Signed-off-by: Pedro Algarvio --- tests/pytests/unit/modules/test_msteams.py | 6 ++++-- tests/pytests/unit/states/test_file.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/pytests/unit/modules/test_msteams.py b/tests/pytests/unit/modules/test_msteams.py index 1307cd854bd..a374dd90321 100644 --- a/tests/pytests/unit/modules/test_msteams.py +++ b/tests/pytests/unit/modules/test_msteams.py @@ -24,8 +24,10 @@ def test_post_card(): with patch("salt.utils.http.query", http_mock): ret = msteams.post_card("test") assert ret - assert http_mock.assert_called_once_with( + http_mock.assert_called_once_with( + "https://example.com/web_hook", method="POST", header_dict={"Content-Type": "application/json"}, - data='{"text": "test", "title": Null, "themeColor": Null}', + data='{"text": "test", "title": null, "themeColor": null}', + status=True, ) diff --git a/tests/pytests/unit/states/test_file.py b/tests/pytests/unit/states/test_file.py index 393ede9b397..03ee53626ee 100644 --- a/tests/pytests/unit/states/test_file.py +++ b/tests/pytests/unit/states/test_file.py @@ -87,6 +87,7 @@ def test_file_recurse_directory_test(): "name": "/tmp/test", "result": False, } - assert salt_dunder["cp.list_master_dirs"].assert_called_once_with( - prefix="does_not_exist/" + salt_dunder["cp.list_master_dirs"].assert_called_once_with( + saltenv="base", + prefix="does_not_exist/", ) From 908732afaa590fe54bac876788c37471c974a3a0 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Thu, 14 Sep 2023 08:05:22 +0100 Subject: [PATCH 23/31] Switch to the correct fixture usage Signed-off-by: Pedro Algarvio --- tests/pytests/unit/states/test_file.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/pytests/unit/states/test_file.py b/tests/pytests/unit/states/test_file.py index 03ee53626ee..b7e239045b4 100644 --- a/tests/pytests/unit/states/test_file.py +++ b/tests/pytests/unit/states/test_file.py @@ -7,19 +7,17 @@ import salt.states.file as file from tests.support.mock import MagicMock, call, create_autospec, patch -@pytest.fixture(autouse=True) -def setup_loader(request): - setup_loader_modules = { +@pytest.fixture +def configure_loader_modules(minion_opts): + return { file: { "__opts__": {"test": False}, "__env__": "base", } } - with pytest.helpers.loader_mock(request, setup_loader_modules) as loader_mock: - yield loader_mock -@pytest.fixture() +@pytest.fixture def fake_remove(): fake_remove_mod = create_autospec(filemod.remove) with patch.dict(file.__salt__, {"file.remove": fake_remove_mod}): From 1981c3784d4f331de6d79b871a3c972b449ff88f Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 3 Oct 2023 17:08:39 +0100 Subject: [PATCH 24/31] Switch to `more-itertools` when splitting the tests into slices/chunks Signed-off-by: Pedro Algarvio --- requirements/pytest.txt | 1 + requirements/static/ci/py3.10/cloud.txt | 1 + requirements/static/ci/py3.10/darwin.txt | 1 + requirements/static/ci/py3.10/freebsd.txt | 1 + requirements/static/ci/py3.10/linux.txt | 1 + requirements/static/ci/py3.10/windows.txt | 1 + requirements/static/ci/py3.7/cloud.txt | 1 + requirements/static/ci/py3.7/freebsd.txt | 1 + requirements/static/ci/py3.7/linux.txt | 1 + requirements/static/ci/py3.7/windows.txt | 1 + requirements/static/ci/py3.8/cloud.txt | 1 + requirements/static/ci/py3.8/freebsd.txt | 1 + requirements/static/ci/py3.8/linux.txt | 1 + requirements/static/ci/py3.8/windows.txt | 1 + requirements/static/ci/py3.9/cloud.txt | 1 + requirements/static/ci/py3.9/darwin.txt | 1 + requirements/static/ci/py3.9/freebsd.txt | 1 + requirements/static/ci/py3.9/linux.txt | 1 + requirements/static/ci/py3.9/windows.txt | 1 + tests/conftest.py | 39 ++++++----------------- 20 files changed, 29 insertions(+), 29 deletions(-) diff --git a/requirements/pytest.txt b/requirements/pytest.txt index 8573ca8a9e8..cafa2ec25a1 100644 --- a/requirements/pytest.txt +++ b/requirements/pytest.txt @@ -9,3 +9,4 @@ pytest-timeout pytest-httpserver pytest-custom-exit-code >= 0.3 flaky +more-itertools diff --git a/requirements/static/ci/py3.10/cloud.txt b/requirements/static/ci/py3.10/cloud.txt index 1179cd38b7d..61690e249bb 100644 --- a/requirements/static/ci/py3.10/cloud.txt +++ b/requirements/static/ci/py3.10/cloud.txt @@ -796,6 +796,7 @@ more-itertools==5.0.0 # via # -c requirements/static/ci/../pkg/py3.10/linux.txt # -c requirements/static/ci/py3.10/linux.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.10/darwin.txt b/requirements/static/ci/py3.10/darwin.txt index 256fe689592..84c3084d76d 100644 --- a/requirements/static/ci/py3.10/darwin.txt +++ b/requirements/static/ci/py3.10/darwin.txt @@ -541,6 +541,7 @@ mock==5.1.0 more-itertools==8.2.0 # via # -c requirements/static/ci/../pkg/py3.10/darwin.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.10/freebsd.txt b/requirements/static/ci/py3.10/freebsd.txt index 18f4f42651c..0fceb5d4745 100644 --- a/requirements/static/ci/py3.10/freebsd.txt +++ b/requirements/static/ci/py3.10/freebsd.txt @@ -533,6 +533,7 @@ mock==5.1.0 more-itertools==5.0.0 # via # -c requirements/static/ci/../pkg/py3.10/freebsd.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.10/linux.txt b/requirements/static/ci/py3.10/linux.txt index 0049950e1ef..64def55e0c7 100644 --- a/requirements/static/ci/py3.10/linux.txt +++ b/requirements/static/ci/py3.10/linux.txt @@ -548,6 +548,7 @@ mock==5.1.0 more-itertools==5.0.0 # via # -c requirements/static/ci/../pkg/py3.10/linux.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.10/windows.txt b/requirements/static/ci/py3.10/windows.txt index 92a4b88303b..2fa1115bdba 100644 --- a/requirements/static/ci/py3.10/windows.txt +++ b/requirements/static/ci/py3.10/windows.txt @@ -213,6 +213,7 @@ mock==5.1.0 more-itertools==8.2.0 # via # -c requirements/static/ci/../pkg/py3.10/windows.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.7/cloud.txt b/requirements/static/ci/py3.7/cloud.txt index 97736d31111..6f2799bc246 100644 --- a/requirements/static/ci/py3.7/cloud.txt +++ b/requirements/static/ci/py3.7/cloud.txt @@ -826,6 +826,7 @@ more-itertools==5.0.0 # via # -c requirements/static/ci/../pkg/py3.7/linux.txt # -c requirements/static/ci/py3.7/linux.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.7/freebsd.txt b/requirements/static/ci/py3.7/freebsd.txt index a979abdfb10..ece9fe46717 100644 --- a/requirements/static/ci/py3.7/freebsd.txt +++ b/requirements/static/ci/py3.7/freebsd.txt @@ -557,6 +557,7 @@ mock==5.1.0 more-itertools==5.0.0 # via # -c requirements/static/ci/../pkg/py3.7/freebsd.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.7/linux.txt b/requirements/static/ci/py3.7/linux.txt index 6f648598b31..cb9b1dd651b 100644 --- a/requirements/static/ci/py3.7/linux.txt +++ b/requirements/static/ci/py3.7/linux.txt @@ -572,6 +572,7 @@ mock==5.1.0 more-itertools==5.0.0 # via # -c requirements/static/ci/../pkg/py3.7/linux.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.7/windows.txt b/requirements/static/ci/py3.7/windows.txt index 3a785589f0d..c1a761916ad 100644 --- a/requirements/static/ci/py3.7/windows.txt +++ b/requirements/static/ci/py3.7/windows.txt @@ -227,6 +227,7 @@ mock==5.1.0 more-itertools==8.2.0 # via # -c requirements/static/ci/../pkg/py3.7/windows.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.8/cloud.txt b/requirements/static/ci/py3.8/cloud.txt index 5c0e53d37a7..fb1e07cb799 100644 --- a/requirements/static/ci/py3.8/cloud.txt +++ b/requirements/static/ci/py3.8/cloud.txt @@ -814,6 +814,7 @@ more-itertools==5.0.0 # via # -c requirements/static/ci/../pkg/py3.8/linux.txt # -c requirements/static/ci/py3.8/linux.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.8/freebsd.txt b/requirements/static/ci/py3.8/freebsd.txt index 0a035c08cdf..40b37ce200b 100644 --- a/requirements/static/ci/py3.8/freebsd.txt +++ b/requirements/static/ci/py3.8/freebsd.txt @@ -547,6 +547,7 @@ mock==5.1.0 more-itertools==5.0.0 # via # -c requirements/static/ci/../pkg/py3.8/freebsd.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.8/linux.txt b/requirements/static/ci/py3.8/linux.txt index 337f491ddc5..b5f42c25008 100644 --- a/requirements/static/ci/py3.8/linux.txt +++ b/requirements/static/ci/py3.8/linux.txt @@ -562,6 +562,7 @@ mock==5.1.0 more-itertools==5.0.0 # via # -c requirements/static/ci/../pkg/py3.8/linux.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.8/windows.txt b/requirements/static/ci/py3.8/windows.txt index 244a1e0734f..95fcf8c0570 100644 --- a/requirements/static/ci/py3.8/windows.txt +++ b/requirements/static/ci/py3.8/windows.txt @@ -215,6 +215,7 @@ mock==5.1.0 more-itertools==8.2.0 # via # -c requirements/static/ci/../pkg/py3.8/windows.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.9/cloud.txt b/requirements/static/ci/py3.9/cloud.txt index 47593bdb4f1..2a60c038d87 100644 --- a/requirements/static/ci/py3.9/cloud.txt +++ b/requirements/static/ci/py3.9/cloud.txt @@ -814,6 +814,7 @@ more-itertools==5.0.0 # via # -c requirements/static/ci/../pkg/py3.9/linux.txt # -c requirements/static/ci/py3.9/linux.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.9/darwin.txt b/requirements/static/ci/py3.9/darwin.txt index 05970bc3186..3983eec0a46 100644 --- a/requirements/static/ci/py3.9/darwin.txt +++ b/requirements/static/ci/py3.9/darwin.txt @@ -555,6 +555,7 @@ mock==5.1.0 more-itertools==8.2.0 # via # -c requirements/static/ci/../pkg/py3.9/darwin.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.9/freebsd.txt b/requirements/static/ci/py3.9/freebsd.txt index 1e75e0a42d8..a503e7130a9 100644 --- a/requirements/static/ci/py3.9/freebsd.txt +++ b/requirements/static/ci/py3.9/freebsd.txt @@ -547,6 +547,7 @@ mock==5.1.0 more-itertools==5.0.0 # via # -c requirements/static/ci/../pkg/py3.9/freebsd.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.9/linux.txt b/requirements/static/ci/py3.9/linux.txt index 8d2c2f6d54d..9add0ba702a 100644 --- a/requirements/static/ci/py3.9/linux.txt +++ b/requirements/static/ci/py3.9/linux.txt @@ -564,6 +564,7 @@ mock==5.1.0 more-itertools==5.0.0 # via # -c requirements/static/ci/../pkg/py3.9/linux.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/requirements/static/ci/py3.9/windows.txt b/requirements/static/ci/py3.9/windows.txt index 74979a411af..5dcd12b602e 100644 --- a/requirements/static/ci/py3.9/windows.txt +++ b/requirements/static/ci/py3.9/windows.txt @@ -215,6 +215,7 @@ mock==5.1.0 more-itertools==8.2.0 # via # -c requirements/static/ci/../pkg/py3.9/windows.txt + # -r requirements/pytest.txt # cheroot # cherrypy # jaraco.classes diff --git a/tests/conftest.py b/tests/conftest.py index 5bb4e5764c3..197ba8433dd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,6 +15,7 @@ from unittest import TestCase # pylint: disable=blacklisted-module import _pytest.logging import _pytest.skipping +import more_itertools import psutil import pytest @@ -800,33 +801,6 @@ def pytest_runtest_setup(item): # ----- Test Groups Selection ---------------------------------------------------------------------------------------> -def get_group_size_and_start(total_items, total_groups, group_id): - """ - Calculate group size and start index. - """ - base_size = total_items // total_groups - rem = total_items % total_groups - - start = base_size * (group_id - 1) + min(group_id - 1, rem) - size = base_size + 1 if group_id <= rem else base_size - - return (start, size) - - -def get_group(items, total_groups, group_id): - """ - Get the items from the passed in group based on group size. - """ - if not 0 < group_id <= total_groups: - raise ValueError("Invalid test-group argument") - - start, size = get_group_size_and_start(len(items), total_groups, group_id) - selected = items[start : start + size] - deselected = items[:start] + items[start + size :] - assert len(selected) + len(deselected) == len(items) - return selected, deselected - - def groups_collection_modifyitems(config, items): group_count = config.getoption("test-group-count") group_id = config.getoption("test-group") @@ -841,7 +815,14 @@ def groups_collection_modifyitems(config, items): total_items = len(items) - tests_in_group, deselected = get_group(items, group_count, group_id) + # Devide into test groups + test_groups = more_itertools.divide(group_count, items) + # Pick the right group + tests_in_group = list(test_groups.pop(group_id - 1)) + # The rest are deselected tests + deselected = list(more_itertools.collapse(test_groups)) + # Sanity check + assert len(tests_in_group) + len(deselected) == total_items # Replace all items in the list items[:] = tests_in_group if deselected: @@ -849,7 +830,7 @@ def groups_collection_modifyitems(config, items): terminal_reporter = config.pluginmanager.get_plugin("terminalreporter") terminal_reporter.write( - f"Running test group #{group_id}(out of #{group_count}) ({len(items)} tests)\n", + f"Running test group #{group_id}(out of #{group_count}) ({len(items)} out of {total_items} tests)\n", yellow=True, ) From 1f9a8e89a54024601edb0096b62abbbae77890fa Mon Sep 17 00:00:00 2001 From: ScriptAutomate Date: Tue, 3 Oct 2023 17:56:49 -0500 Subject: [PATCH 25/31] Fix CI/CD with latest available golden images --- cicd/amis.yml | 2 +- cicd/golden-images.json | 164 ++++++++++++++++++++-------------------- 2 files changed, 83 insertions(+), 83 deletions(-) diff --git a/cicd/amis.yml b/cicd/amis.yml index efa0bef8829..3b206cb0cbf 100644 --- a/cicd/amis.yml +++ b/cicd/amis.yml @@ -1 +1 @@ -centosstream-9-x86_64: ami-091986d83f4c0bdd7 +centosstream-9-x86_64: ami-09b72b340acb62c73 diff --git a/cicd/golden-images.json b/cicd/golden-images.json index d4903742495..d56419485c4 100644 --- a/cicd/golden-images.json +++ b/cicd/golden-images.json @@ -1,8 +1,8 @@ { "almalinux-8-arm64": { - "ami": "ami-09017a2c26bb6cf37", + "ami": "ami-0c5e8a0573bb547d0", "ami_description": "CI Image of AlmaLinux 8 arm64", - "ami_name": "salt-project/ci/almalinux/8/arm64/20230912.1532", + "ami_name": "salt-project/ci/almalinux/8/arm64/20231003.1815", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -10,9 +10,9 @@ "ssh_username": "ec2-user" }, "almalinux-8": { - "ami": "ami-0da7449d7f17dca6d", + "ami": "ami-0575f7f2a015ab1ab", "ami_description": "CI Image of AlmaLinux 8 x86_64", - "ami_name": "salt-project/ci/almalinux/8/x86_64/20230912.1532", + "ami_name": "salt-project/ci/almalinux/8/x86_64/20231003.1815", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -20,9 +20,9 @@ "ssh_username": "ec2-user" }, "almalinux-9-arm64": { - "ami": "ami-0b45894ce343176b0", + "ami": "ami-04563f34c07df6b37", "ami_description": "CI Image of AlmaLinux 9 arm64", - "ami_name": "salt-project/ci/almalinux/9/arm64/20230912.1532", + "ami_name": "salt-project/ci/almalinux/9/arm64/20231003.1815", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -30,9 +30,9 @@ "ssh_username": "ec2-user" }, "almalinux-9": { - "ami": "ami-0c8a554820c140d45", + "ami": "ami-00c51c0a91489a9c5", "ami_description": "CI Image of AlmaLinux 9 x86_64", - "ami_name": "salt-project/ci/almalinux/9/x86_64/20230912.1532", + "ami_name": "salt-project/ci/almalinux/9/x86_64/20231003.1815", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -40,9 +40,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2-arm64": { - "ami": "ami-0aac44852e96fb156", + "ami": "ami-0ee09c7f2bab65079", "ami_description": "CI Image of AmazonLinux 2 arm64", - "ami_name": "salt-project/ci/amazonlinux/2/arm64/20230912.1532", + "ami_name": "salt-project/ci/amazonlinux/2/arm64/20231003.1833", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -50,9 +50,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2": { - "ami": "ami-04bf06c280f2957e0", + "ami": "ami-09db0feda451d650e", "ami_description": "CI Image of AmazonLinux 2 x86_64", - "ami_name": "salt-project/ci/amazonlinux/2/x86_64/20230912.1532", + "ami_name": "salt-project/ci/amazonlinux/2/x86_64/20231003.1833", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -60,9 +60,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2023.1-arm64": { - "ami": "ami-0430562e1dc073734", + "ami": "ami-0c822de48d792fb9d", "ami_description": "CI Image of AmazonLinux 2023.1 arm64", - "ami_name": "salt-project/ci/amazonlinux/2023.1/arm64/20230912.1532", + "ami_name": "salt-project/ci/amazonlinux/2023.1/arm64/20231003.1816", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -70,9 +70,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2023.1": { - "ami": "ami-09b7ab6b9bb5ceca3", + "ami": "ami-03c3c0511beff576d", "ami_description": "CI Image of AmazonLinux 2023.1 x86_64", - "ami_name": "salt-project/ci/amazonlinux/2023.1/x86_64/20230912.1532", + "ami_name": "salt-project/ci/amazonlinux/2023.1/x86_64/20231003.1816", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -80,9 +80,9 @@ "ssh_username": "ec2-user" }, "archlinux-lts": { - "ami": "ami-08ecb254b10e24bca", + "ami": "ami-05f4292e8255b141b", "ami_description": "CI Image of ArchLinux lts x86_64", - "ami_name": "salt-project/ci/archlinux/lts/x86_64/20230912.1532", + "ami_name": "salt-project/ci/archlinux/lts/x86_64/20231003.1816", "arch": "x86_64", "cloudwatch-agent-available": "false", "instance_type": "t3a.large", @@ -90,9 +90,9 @@ "ssh_username": "arch" }, "centos-7-arm64": { - "ami": "ami-0facb0416e994c2d4", + "ami": "ami-025673b9c304d342b", "ami_description": "CI Image of CentOS 7 arm64", - "ami_name": "salt-project/ci/centos/7/arm64/20230912.1553", + "ami_name": "salt-project/ci/centos/7/arm64/20231003.1816", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -100,9 +100,9 @@ "ssh_username": "centos" }, "centos-7": { - "ami": "ami-04e695ebbac38868e", + "ami": "ami-0838321a764f228c7", "ami_description": "CI Image of CentOS 7 x86_64", - "ami_name": "salt-project/ci/centos/7/x86_64/20230912.1553", + "ami_name": "salt-project/ci/centos/7/x86_64/20231003.1816", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -110,9 +110,9 @@ "ssh_username": "centos" }, "centosstream-8-arm64": { - "ami": "ami-02b1e24269822d3fc", + "ami": "ami-091cb758decb487e8", "ami_description": "CI Image of CentOSStream 8 arm64", - "ami_name": "salt-project/ci/centosstream/8/arm64/20230912.1532", + "ami_name": "salt-project/ci/centosstream/8/arm64/20231003.1833", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -120,9 +120,9 @@ "ssh_username": "centos" }, "centosstream-8": { - "ami": "ami-0706ab643607236c0", + "ami": "ami-0fe8dfc4c2f058f18", "ami_description": "CI Image of CentOSStream 8 x86_64", - "ami_name": "salt-project/ci/centosstream/8/x86_64/20230912.1532", + "ami_name": "salt-project/ci/centosstream/8/x86_64/20231003.1833", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -130,9 +130,9 @@ "ssh_username": "centos" }, "centosstream-9-arm64": { - "ami": "ami-032abe3ace927c296", + "ami": "ami-047b28f087ea4ff2d", "ami_description": "CI Image of CentOSStream 9 arm64", - "ami_name": "salt-project/ci/centosstream/9/arm64/20230912.1532", + "ami_name": "salt-project/ci/centosstream/9/arm64/20231003.1833", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -140,9 +140,9 @@ "ssh_username": "ec2-user" }, "centosstream-9": { - "ami": "ami-091986d83f4c0bdd7", + "ami": "ami-09b72b340acb62c73", "ami_description": "CI Image of CentOSStream 9 x86_64", - "ami_name": "salt-project/ci/centosstream/9/x86_64/20230912.1532", + "ami_name": "salt-project/ci/centosstream/9/x86_64/20231003.1833", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -150,9 +150,9 @@ "ssh_username": "ec2-user" }, "debian-10-arm64": { - "ami": "ami-08b7b9fb74d7c58f2", + "ami": "ami-000a9eea8161d49c2", "ami_description": "CI Image of Debian 10 arm64", - "ami_name": "salt-project/ci/debian/10/arm64/20230912.1546", + "ami_name": "salt-project/ci/debian/10/arm64/20231003.1816", "arch": "arm64", "cloudwatch-agent-available": "false", "instance_type": "m6g.large", @@ -160,9 +160,9 @@ "ssh_username": "admin" }, "debian-10": { - "ami": "ami-0002ea04be195948e", + "ami": "ami-0bc86919c717b283f", "ami_description": "CI Image of Debian 10 x86_64", - "ami_name": "salt-project/ci/debian/10/x86_64/20230912.1548", + "ami_name": "salt-project/ci/debian/10/x86_64/20231003.1816", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -170,9 +170,9 @@ "ssh_username": "admin" }, "debian-11-arm64": { - "ami": "ami-0e14ec1b2a5553f96", + "ami": "ami-0d8a43c4b48875d5e", "ami_description": "CI Image of Debian 11 arm64", - "ami_name": "salt-project/ci/debian/11/arm64/20230912.1548", + "ami_name": "salt-project/ci/debian/11/arm64/20231003.1816", "arch": "arm64", "cloudwatch-agent-available": "false", "instance_type": "m6g.large", @@ -180,9 +180,9 @@ "ssh_username": "admin" }, "debian-11": { - "ami": "ami-06c5ea0d19a5773d7", + "ami": "ami-08f2aa9302ed94d03", "ami_description": "CI Image of Debian 11 x86_64", - "ami_name": "salt-project/ci/debian/11/x86_64/20230912.1549", + "ami_name": "salt-project/ci/debian/11/x86_64/20231003.1816", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -190,9 +190,9 @@ "ssh_username": "admin" }, "debian-12-arm64": { - "ami": "ami-055b0a6d5bb3e9ecd", + "ami": "ami-040167a351bab697a", "ami_description": "CI Image of Debian 12 arm64", - "ami_name": "salt-project/ci/debian/12/arm64/20230912.1550", + "ami_name": "salt-project/ci/debian/12/arm64/20231003.1816", "arch": "arm64", "cloudwatch-agent-available": "false", "instance_type": "m6g.large", @@ -200,9 +200,9 @@ "ssh_username": "admin" }, "debian-12": { - "ami": "ami-0eada119571a913fd", + "ami": "ami-0ab900bcd009184e1", "ami_description": "CI Image of Debian 12 x86_64", - "ami_name": "salt-project/ci/debian/12/x86_64/20230912.1550", + "ami_name": "salt-project/ci/debian/12/x86_64/20231003.1816", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -210,9 +210,9 @@ "ssh_username": "admin" }, "fedora-37-arm64": { - "ami": "ami-02b916b21581ead5a", + "ami": "ami-0b1384d087e911992", "ami_description": "CI Image of Fedora 37 arm64", - "ami_name": "salt-project/ci/fedora/37/arm64/20230912.1533", + "ami_name": "salt-project/ci/fedora/37/arm64/20231003.1815", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -220,9 +220,9 @@ "ssh_username": "fedora" }, "fedora-37": { - "ami": "ami-01e0becc3552ad2f6", + "ami": "ami-0b6ddef61ab49e347", "ami_description": "CI Image of Fedora 37 x86_64", - "ami_name": "salt-project/ci/fedora/37/x86_64/20230912.1533", + "ami_name": "salt-project/ci/fedora/37/x86_64/20231003.1815", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -230,9 +230,9 @@ "ssh_username": "fedora" }, "fedora-38-arm64": { - "ami": "ami-01f4a3bdee88da9f3", + "ami": "ami-06fbccbb7128d2ab0", "ami_description": "CI Image of Fedora 38 arm64", - "ami_name": "salt-project/ci/fedora/38/arm64/20230912.1533", + "ami_name": "salt-project/ci/fedora/38/arm64/20231003.1815", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -240,9 +240,9 @@ "ssh_username": "fedora" }, "fedora-38": { - "ami": "ami-0c38a1907f5b55077", + "ami": "ami-0a2ac0bdbbcf16da9", "ami_description": "CI Image of Fedora 38 x86_64", - "ami_name": "salt-project/ci/fedora/38/x86_64/20230912.1543", + "ami_name": "salt-project/ci/fedora/38/x86_64/20231003.1815", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -250,9 +250,9 @@ "ssh_username": "fedora" }, "opensuse-15": { - "ami": "ami-0ecfb817deee506a9", + "ami": "ami-0f371e1a07228e0b4", "ami_description": "CI Image of Opensuse 15 x86_64", - "ami_name": "salt-project/ci/opensuse/15/x86_64/20230912.1533", + "ami_name": "salt-project/ci/opensuse/15/x86_64/20231003.1816", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -260,9 +260,9 @@ "ssh_username": "ec2-user" }, "photonos-3-arm64": { - "ami": "ami-0383031c08217b13e", + "ami": "ami-07561a2fff97ceb7e", "ami_description": "CI Image of PhotonOS 3 arm64", - "ami_name": "salt-project/ci/photonos/3/arm64/20230924.0913", + "ami_name": "salt-project/ci/photonos/3/arm64/20231003.1833", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -270,9 +270,9 @@ "ssh_username": "root" }, "photonos-3": { - "ami": "ami-06004a7d856e94355", + "ami": "ami-0a59b9fc8ef4c0839", "ami_description": "CI Image of PhotonOS 3 x86_64", - "ami_name": "salt-project/ci/photonos/3/x86_64/20230924.0913", + "ami_name": "salt-project/ci/photonos/3/x86_64/20231003.1834", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -280,9 +280,9 @@ "ssh_username": "root" }, "photonos-4-arm64": { - "ami": "ami-08808bcf97f824036", + "ami": "ami-0f03b431b8da281f4", "ami_description": "CI Image of PhotonOS 4 arm64", - "ami_name": "salt-project/ci/photonos/4/arm64/20230924.0924", + "ami_name": "salt-project/ci/photonos/4/arm64/20231003.1834", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -290,9 +290,9 @@ "ssh_username": "root" }, "photonos-4": { - "ami": "ami-0fd7a6ed4c61ee312", + "ami": "ami-0889afb0d3f3b5d0e", "ami_description": "CI Image of PhotonOS 4 x86_64", - "ami_name": "salt-project/ci/photonos/4/x86_64/20230924.0925", + "ami_name": "salt-project/ci/photonos/4/x86_64/20231003.1834", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -300,9 +300,9 @@ "ssh_username": "root" }, "photonos-5-arm64": { - "ami": "ami-0c85ad72d107ec4b8", + "ami": "ami-07442a02f7fe6b718", "ami_description": "CI Image of PhotonOS 5 arm64", - "ami_name": "salt-project/ci/photonos/5/arm64/20230924.0927", + "ami_name": "salt-project/ci/photonos/5/arm64/20231003.1834", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -310,9 +310,9 @@ "ssh_username": "root" }, "photonos-5": { - "ami": "ami-0732ce03b2ab6fad2", + "ami": "ami-086f2340a51cb9c3f", "ami_description": "CI Image of PhotonOS 5 x86_64", - "ami_name": "salt-project/ci/photonos/5/x86_64/20230924.0927", + "ami_name": "salt-project/ci/photonos/5/x86_64/20231003.1835", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -320,9 +320,9 @@ "ssh_username": "root" }, "ubuntu-20.04-arm64": { - "ami": "ami-05aeb3d5bf0a16369", + "ami": "ami-033e27e2d03cf3d10", "ami_description": "CI Image of Ubuntu 20.04 arm64", - "ami_name": "salt-project/ci/ubuntu/20.04/arm64/20230912.1551", + "ami_name": "salt-project/ci/ubuntu/20.04/arm64/20231003.1816", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -330,9 +330,9 @@ "ssh_username": "ubuntu" }, "ubuntu-20.04": { - "ami": "ami-00cec0054fd71d281", + "ami": "ami-0e878775d8e2a746d", "ami_description": "CI Image of Ubuntu 20.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/20.04/x86_64/20230912.1550", + "ami_name": "salt-project/ci/ubuntu/20.04/x86_64/20231003.1816", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -340,9 +340,9 @@ "ssh_username": "ubuntu" }, "ubuntu-22.04-arm64": { - "ami": "ami-0f7dc3333620d58fd", + "ami": "ami-01be9d89d8cbf09de", "ami_description": "CI Image of Ubuntu 22.04 arm64", - "ami_name": "salt-project/ci/ubuntu/22.04/arm64/20230912.1551", + "ami_name": "salt-project/ci/ubuntu/22.04/arm64/20231003.1816", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -350,9 +350,9 @@ "ssh_username": "ubuntu" }, "ubuntu-22.04": { - "ami": "ami-0bc7c1824a6b0752f", + "ami": "ami-0af23aeeab0dc5c31", "ami_description": "CI Image of Ubuntu 22.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/22.04/x86_64/20230912.1552", + "ami_name": "salt-project/ci/ubuntu/22.04/x86_64/20231003.1816", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -360,9 +360,9 @@ "ssh_username": "ubuntu" }, "ubuntu-23.04-arm64": { - "ami": "ami-0e8818777218efeeb", + "ami": "ami-0afa9a2fdcc3d6479", "ami_description": "CI Image of Ubuntu 23.04 arm64", - "ami_name": "salt-project/ci/ubuntu/23.04/arm64/20230912.1552", + "ami_name": "salt-project/ci/ubuntu/23.04/arm64/20231003.1816", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -370,9 +370,9 @@ "ssh_username": "ubuntu" }, "ubuntu-23.04": { - "ami": "ami-0813a38bf6a6cf4de", + "ami": "ami-05990aaa92c7aa77f", "ami_description": "CI Image of Ubuntu 23.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/23.04/x86_64/20230912.1552", + "ami_name": "salt-project/ci/ubuntu/23.04/x86_64/20231003.1816", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -380,9 +380,9 @@ "ssh_username": "ubuntu" }, "windows-2016": { - "ami": "ami-099db55543619f54a", + "ami": "ami-0351b17103b653e06", "ami_description": "CI Image of Windows 2016 x86_64", - "ami_name": "salt-project/ci/windows/2016/x86_64/20230522.0606", + "ami_name": "salt-project/ci/windows/2016/x86_64/20231003.1831", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", @@ -390,9 +390,9 @@ "ssh_username": "Administrator" }, "windows-2019": { - "ami": "ami-0860ee5bc9ee93e13", + "ami": "ami-027a53f66275e2fcd", "ami_description": "CI Image of Windows 2019 x86_64", - "ami_name": "salt-project/ci/windows/2019/x86_64/20230522.0606", + "ami_name": "salt-project/ci/windows/2019/x86_64/20231003.1832", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", @@ -400,9 +400,9 @@ "ssh_username": "Administrator" }, "windows-2022": { - "ami": "ami-032e3abce2aa98da7", + "ami": "ami-09ddba69a43e5f0ed", "ami_description": "CI Image of Windows 2022 x86_64", - "ami_name": "salt-project/ci/windows/2022/x86_64/20230522.0606", + "ami_name": "salt-project/ci/windows/2022/x86_64/20231003.1922", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", From dfd5194dd05776575521fd0fad5e913111f122af Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 4 Oct 2023 13:20:29 +0100 Subject: [PATCH 26/31] De-whitelist, again, since it's hanging on the newer windows golden images Signed-off-by: Pedro Algarvio --- tests/integration/modules/test_ssh.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/modules/test_ssh.py b/tests/integration/modules/test_ssh.py index 0817877c86b..55586211622 100644 --- a/tests/integration/modules/test_ssh.py +++ b/tests/integration/modules/test_ssh.py @@ -26,7 +26,8 @@ def check_status(): return False -@pytest.mark.windows_whitelisted +# @pytest.mark.windows_whitelisted +# De-whitelist windows since it's hanging on the newer windows golden images @pytest.mark.skip_if_binaries_missing("ssh", "ssh-keygen", check_all=True) class SSHModuleTest(ModuleCase): """ From 6fb7f395d4ddd7158fe88e4ad3e3086ce1c54a56 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 4 Oct 2023 13:46:43 +0100 Subject: [PATCH 27/31] Update to latest golden images Signed-off-by: Pedro Algarvio --- cicd/golden-images.json | 164 ++++++++++++++++++++-------------------- 1 file changed, 82 insertions(+), 82 deletions(-) diff --git a/cicd/golden-images.json b/cicd/golden-images.json index d56419485c4..9a3f06f705a 100644 --- a/cicd/golden-images.json +++ b/cicd/golden-images.json @@ -1,8 +1,8 @@ { "almalinux-8-arm64": { - "ami": "ami-0c5e8a0573bb547d0", + "ami": "ami-0f08fc00f1689a8ec", "ami_description": "CI Image of AlmaLinux 8 arm64", - "ami_name": "salt-project/ci/almalinux/8/arm64/20231003.1815", + "ami_name": "salt-project/ci/almalinux/8/arm64/20231003.2057", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -10,9 +10,9 @@ "ssh_username": "ec2-user" }, "almalinux-8": { - "ami": "ami-0575f7f2a015ab1ab", + "ami": "ami-08f648e0e6fa619c2", "ami_description": "CI Image of AlmaLinux 8 x86_64", - "ami_name": "salt-project/ci/almalinux/8/x86_64/20231003.1815", + "ami_name": "salt-project/ci/almalinux/8/x86_64/20231003.2058", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -20,9 +20,9 @@ "ssh_username": "ec2-user" }, "almalinux-9-arm64": { - "ami": "ami-04563f34c07df6b37", + "ami": "ami-0394b210e1e09b962", "ami_description": "CI Image of AlmaLinux 9 arm64", - "ami_name": "salt-project/ci/almalinux/9/arm64/20231003.1815", + "ami_name": "salt-project/ci/almalinux/9/arm64/20231003.2058", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -30,9 +30,9 @@ "ssh_username": "ec2-user" }, "almalinux-9": { - "ami": "ami-00c51c0a91489a9c5", + "ami": "ami-0a909a150cfebea5b", "ami_description": "CI Image of AlmaLinux 9 x86_64", - "ami_name": "salt-project/ci/almalinux/9/x86_64/20231003.1815", + "ami_name": "salt-project/ci/almalinux/9/x86_64/20231003.2100", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -40,9 +40,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2-arm64": { - "ami": "ami-0ee09c7f2bab65079", + "ami": "ami-038eac6a08feecdb2", "ami_description": "CI Image of AmazonLinux 2 arm64", - "ami_name": "salt-project/ci/amazonlinux/2/arm64/20231003.1833", + "ami_name": "salt-project/ci/amazonlinux/2/arm64/20231003.2104", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -50,9 +50,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2": { - "ami": "ami-09db0feda451d650e", + "ami": "ami-09682e96e7785642d", "ami_description": "CI Image of AmazonLinux 2 x86_64", - "ami_name": "salt-project/ci/amazonlinux/2/x86_64/20231003.1833", + "ami_name": "salt-project/ci/amazonlinux/2/x86_64/20231003.2104", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -60,9 +60,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2023.1-arm64": { - "ami": "ami-0c822de48d792fb9d", + "ami": "ami-0e46c84fb43817334", "ami_description": "CI Image of AmazonLinux 2023.1 arm64", - "ami_name": "salt-project/ci/amazonlinux/2023.1/arm64/20231003.1816", + "ami_name": "salt-project/ci/amazonlinux/2023.1/arm64/20231003.2103", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -70,9 +70,9 @@ "ssh_username": "ec2-user" }, "amazonlinux-2023.1": { - "ami": "ami-03c3c0511beff576d", + "ami": "ami-0ac591368ec230345", "ami_description": "CI Image of AmazonLinux 2023.1 x86_64", - "ami_name": "salt-project/ci/amazonlinux/2023.1/x86_64/20231003.1816", + "ami_name": "salt-project/ci/amazonlinux/2023.1/x86_64/20231003.2103", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -80,9 +80,9 @@ "ssh_username": "ec2-user" }, "archlinux-lts": { - "ami": "ami-05f4292e8255b141b", + "ami": "ami-017de6f1e636021a0", "ami_description": "CI Image of ArchLinux lts x86_64", - "ami_name": "salt-project/ci/archlinux/lts/x86_64/20231003.1816", + "ami_name": "salt-project/ci/archlinux/lts/x86_64/20231003.2108", "arch": "x86_64", "cloudwatch-agent-available": "false", "instance_type": "t3a.large", @@ -90,9 +90,9 @@ "ssh_username": "arch" }, "centos-7-arm64": { - "ami": "ami-025673b9c304d342b", + "ami": "ami-088cb5f3066efa748", "ami_description": "CI Image of CentOS 7 arm64", - "ami_name": "salt-project/ci/centos/7/arm64/20231003.1816", + "ami_name": "salt-project/ci/centos/7/arm64/20231003.2108", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -100,9 +100,9 @@ "ssh_username": "centos" }, "centos-7": { - "ami": "ami-0838321a764f228c7", + "ami": "ami-05c4056c36cecc136", "ami_description": "CI Image of CentOS 7 x86_64", - "ami_name": "salt-project/ci/centos/7/x86_64/20231003.1816", + "ami_name": "salt-project/ci/centos/7/x86_64/20231003.2107", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -110,9 +110,9 @@ "ssh_username": "centos" }, "centosstream-8-arm64": { - "ami": "ami-091cb758decb487e8", + "ami": "ami-0e2a761782490f7c2", "ami_description": "CI Image of CentOSStream 8 arm64", - "ami_name": "salt-project/ci/centosstream/8/arm64/20231003.1833", + "ami_name": "salt-project/ci/centosstream/8/arm64/20231003.2109", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -120,9 +120,9 @@ "ssh_username": "centos" }, "centosstream-8": { - "ami": "ami-0fe8dfc4c2f058f18", + "ami": "ami-06178cd094ea71c34", "ami_description": "CI Image of CentOSStream 8 x86_64", - "ami_name": "salt-project/ci/centosstream/8/x86_64/20231003.1833", + "ami_name": "salt-project/ci/centosstream/8/x86_64/20231003.2108", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -130,9 +130,9 @@ "ssh_username": "centos" }, "centosstream-9-arm64": { - "ami": "ami-047b28f087ea4ff2d", + "ami": "ami-0ea1025028e6fe700", "ami_description": "CI Image of CentOSStream 9 arm64", - "ami_name": "salt-project/ci/centosstream/9/arm64/20231003.1833", + "ami_name": "salt-project/ci/centosstream/9/arm64/20231003.2109", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -140,9 +140,9 @@ "ssh_username": "ec2-user" }, "centosstream-9": { - "ami": "ami-09b72b340acb62c73", + "ami": "ami-0f474b360fca72512", "ami_description": "CI Image of CentOSStream 9 x86_64", - "ami_name": "salt-project/ci/centosstream/9/x86_64/20231003.1833", + "ami_name": "salt-project/ci/centosstream/9/x86_64/20231003.2109", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -150,9 +150,9 @@ "ssh_username": "ec2-user" }, "debian-10-arm64": { - "ami": "ami-000a9eea8161d49c2", + "ami": "ami-0b9cbee875ae2e145", "ami_description": "CI Image of Debian 10 arm64", - "ami_name": "salt-project/ci/debian/10/arm64/20231003.1816", + "ami_name": "salt-project/ci/debian/10/arm64/20231003.2114", "arch": "arm64", "cloudwatch-agent-available": "false", "instance_type": "m6g.large", @@ -160,9 +160,9 @@ "ssh_username": "admin" }, "debian-10": { - "ami": "ami-0bc86919c717b283f", + "ami": "ami-03b713e88ac915c18", "ami_description": "CI Image of Debian 10 x86_64", - "ami_name": "salt-project/ci/debian/10/x86_64/20231003.1816", + "ami_name": "salt-project/ci/debian/10/x86_64/20231003.2112", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -170,9 +170,9 @@ "ssh_username": "admin" }, "debian-11-arm64": { - "ami": "ami-0d8a43c4b48875d5e", + "ami": "ami-0e48f24d9def8d84c", "ami_description": "CI Image of Debian 11 arm64", - "ami_name": "salt-project/ci/debian/11/arm64/20231003.1816", + "ami_name": "salt-project/ci/debian/11/arm64/20231003.2114", "arch": "arm64", "cloudwatch-agent-available": "false", "instance_type": "m6g.large", @@ -180,9 +180,9 @@ "ssh_username": "admin" }, "debian-11": { - "ami": "ami-08f2aa9302ed94d03", + "ami": "ami-07a2fb75d29d0d6f7", "ami_description": "CI Image of Debian 11 x86_64", - "ami_name": "salt-project/ci/debian/11/x86_64/20231003.1816", + "ami_name": "salt-project/ci/debian/11/x86_64/20231003.2116", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -190,9 +190,9 @@ "ssh_username": "admin" }, "debian-12-arm64": { - "ami": "ami-040167a351bab697a", + "ami": "ami-027199ded9ce9f659", "ami_description": "CI Image of Debian 12 arm64", - "ami_name": "salt-project/ci/debian/12/arm64/20231003.1816", + "ami_name": "salt-project/ci/debian/12/arm64/20231003.2117", "arch": "arm64", "cloudwatch-agent-available": "false", "instance_type": "m6g.large", @@ -200,9 +200,9 @@ "ssh_username": "admin" }, "debian-12": { - "ami": "ami-0ab900bcd009184e1", + "ami": "ami-02156ad853a403599", "ami_description": "CI Image of Debian 12 x86_64", - "ami_name": "salt-project/ci/debian/12/x86_64/20231003.1816", + "ami_name": "salt-project/ci/debian/12/x86_64/20231003.2119", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -210,9 +210,9 @@ "ssh_username": "admin" }, "fedora-37-arm64": { - "ami": "ami-0b1384d087e911992", + "ami": "ami-0dfb1b2e3b6cd8847", "ami_description": "CI Image of Fedora 37 arm64", - "ami_name": "salt-project/ci/fedora/37/arm64/20231003.1815", + "ami_name": "salt-project/ci/fedora/37/arm64/20231003.2119", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -220,9 +220,9 @@ "ssh_username": "fedora" }, "fedora-37": { - "ami": "ami-0b6ddef61ab49e347", + "ami": "ami-0d27e014bf07af18b", "ami_description": "CI Image of Fedora 37 x86_64", - "ami_name": "salt-project/ci/fedora/37/x86_64/20231003.1815", + "ami_name": "salt-project/ci/fedora/37/x86_64/20231003.2120", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -230,9 +230,9 @@ "ssh_username": "fedora" }, "fedora-38-arm64": { - "ami": "ami-06fbccbb7128d2ab0", + "ami": "ami-04f5a34bae3040974", "ami_description": "CI Image of Fedora 38 arm64", - "ami_name": "salt-project/ci/fedora/38/arm64/20231003.1815", + "ami_name": "salt-project/ci/fedora/38/arm64/20231003.2120", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -240,9 +240,9 @@ "ssh_username": "fedora" }, "fedora-38": { - "ami": "ami-0a2ac0bdbbcf16da9", + "ami": "ami-0e69802061ed79891", "ami_description": "CI Image of Fedora 38 x86_64", - "ami_name": "salt-project/ci/fedora/38/x86_64/20231003.1815", + "ami_name": "salt-project/ci/fedora/38/x86_64/20231003.2123", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -250,9 +250,9 @@ "ssh_username": "fedora" }, "opensuse-15": { - "ami": "ami-0f371e1a07228e0b4", + "ami": "ami-0ebb684e16914ad0a", "ami_description": "CI Image of Opensuse 15 x86_64", - "ami_name": "salt-project/ci/opensuse/15/x86_64/20231003.1816", + "ami_name": "salt-project/ci/opensuse/15/x86_64/20231003.2110", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -260,9 +260,9 @@ "ssh_username": "ec2-user" }, "photonos-3-arm64": { - "ami": "ami-07561a2fff97ceb7e", + "ami": "ami-054765b3beb6dd97c", "ami_description": "CI Image of PhotonOS 3 arm64", - "ami_name": "salt-project/ci/photonos/3/arm64/20231003.1833", + "ami_name": "salt-project/ci/photonos/3/arm64/20231003.2129", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -270,9 +270,9 @@ "ssh_username": "root" }, "photonos-3": { - "ami": "ami-0a59b9fc8ef4c0839", + "ami": "ami-0224e8a4471113ebb", "ami_description": "CI Image of PhotonOS 3 x86_64", - "ami_name": "salt-project/ci/photonos/3/x86_64/20231003.1834", + "ami_name": "salt-project/ci/photonos/3/x86_64/20231003.2128", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -280,9 +280,9 @@ "ssh_username": "root" }, "photonos-4-arm64": { - "ami": "ami-0f03b431b8da281f4", + "ami": "ami-091f6d77aa3921394", "ami_description": "CI Image of PhotonOS 4 arm64", - "ami_name": "salt-project/ci/photonos/4/arm64/20231003.1834", + "ami_name": "salt-project/ci/photonos/4/arm64/20231003.2124", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -290,9 +290,9 @@ "ssh_username": "root" }, "photonos-4": { - "ami": "ami-0889afb0d3f3b5d0e", + "ami": "ami-0714704e9471a8e0c", "ami_description": "CI Image of PhotonOS 4 x86_64", - "ami_name": "salt-project/ci/photonos/4/x86_64/20231003.1834", + "ami_name": "salt-project/ci/photonos/4/x86_64/20231003.2130", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -300,9 +300,9 @@ "ssh_username": "root" }, "photonos-5-arm64": { - "ami": "ami-07442a02f7fe6b718", + "ami": "ami-05ebc5bddb487c20b", "ami_description": "CI Image of PhotonOS 5 arm64", - "ami_name": "salt-project/ci/photonos/5/arm64/20231003.1834", + "ami_name": "salt-project/ci/photonos/5/arm64/20231003.2130", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -310,9 +310,9 @@ "ssh_username": "root" }, "photonos-5": { - "ami": "ami-086f2340a51cb9c3f", + "ami": "ami-0b7e17bc1990da3af", "ami_description": "CI Image of PhotonOS 5 x86_64", - "ami_name": "salt-project/ci/photonos/5/x86_64/20231003.1835", + "ami_name": "salt-project/ci/photonos/5/x86_64/20231003.2131", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -320,9 +320,9 @@ "ssh_username": "root" }, "ubuntu-20.04-arm64": { - "ami": "ami-033e27e2d03cf3d10", + "ami": "ami-09210544c9163df86", "ami_description": "CI Image of Ubuntu 20.04 arm64", - "ami_name": "salt-project/ci/ubuntu/20.04/arm64/20231003.1816", + "ami_name": "salt-project/ci/ubuntu/20.04/arm64/20231003.2110", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -330,9 +330,9 @@ "ssh_username": "ubuntu" }, "ubuntu-20.04": { - "ami": "ami-0e878775d8e2a746d", + "ami": "ami-05894335447f4c052", "ami_description": "CI Image of Ubuntu 20.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/20.04/x86_64/20231003.1816", + "ami_name": "salt-project/ci/ubuntu/20.04/x86_64/20231003.2110", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -340,9 +340,9 @@ "ssh_username": "ubuntu" }, "ubuntu-22.04-arm64": { - "ami": "ami-01be9d89d8cbf09de", + "ami": "ami-090423dbe605f6d3e", "ami_description": "CI Image of Ubuntu 22.04 arm64", - "ami_name": "salt-project/ci/ubuntu/22.04/arm64/20231003.1816", + "ami_name": "salt-project/ci/ubuntu/22.04/arm64/20231003.2111", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -350,9 +350,9 @@ "ssh_username": "ubuntu" }, "ubuntu-22.04": { - "ami": "ami-0af23aeeab0dc5c31", + "ami": "ami-0a465357b34ea7fdc", "ami_description": "CI Image of Ubuntu 22.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/22.04/x86_64/20231003.1816", + "ami_name": "salt-project/ci/ubuntu/22.04/x86_64/20231003.2111", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -360,9 +360,9 @@ "ssh_username": "ubuntu" }, "ubuntu-23.04-arm64": { - "ami": "ami-0afa9a2fdcc3d6479", + "ami": "ami-0ed81524d646f95ee", "ami_description": "CI Image of Ubuntu 23.04 arm64", - "ami_name": "salt-project/ci/ubuntu/23.04/arm64/20231003.1816", + "ami_name": "salt-project/ci/ubuntu/23.04/arm64/20231003.2111", "arch": "arm64", "cloudwatch-agent-available": "true", "instance_type": "m6g.large", @@ -370,9 +370,9 @@ "ssh_username": "ubuntu" }, "ubuntu-23.04": { - "ami": "ami-05990aaa92c7aa77f", + "ami": "ami-02c7edd6357be51b6", "ami_description": "CI Image of Ubuntu 23.04 x86_64", - "ami_name": "salt-project/ci/ubuntu/23.04/x86_64/20231003.1816", + "ami_name": "salt-project/ci/ubuntu/23.04/x86_64/20231003.2112", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.large", @@ -380,9 +380,9 @@ "ssh_username": "ubuntu" }, "windows-2016": { - "ami": "ami-0351b17103b653e06", + "ami": "ami-04f113ff291a8953f", "ami_description": "CI Image of Windows 2016 x86_64", - "ami_name": "salt-project/ci/windows/2016/x86_64/20231003.1831", + "ami_name": "salt-project/ci/windows/2016/x86_64/20231003.2104", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", @@ -390,9 +390,9 @@ "ssh_username": "Administrator" }, "windows-2019": { - "ami": "ami-027a53f66275e2fcd", + "ami": "ami-06475f495e0151fc9", "ami_description": "CI Image of Windows 2019 x86_64", - "ami_name": "salt-project/ci/windows/2019/x86_64/20231003.1832", + "ami_name": "salt-project/ci/windows/2019/x86_64/20231003.2106", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", @@ -400,9 +400,9 @@ "ssh_username": "Administrator" }, "windows-2022": { - "ami": "ami-09ddba69a43e5f0ed", + "ami": "ami-0558da89560480f32", "ami_description": "CI Image of Windows 2022 x86_64", - "ami_name": "salt-project/ci/windows/2022/x86_64/20231003.1922", + "ami_name": "salt-project/ci/windows/2022/x86_64/20231003.2106", "arch": "x86_64", "cloudwatch-agent-available": "true", "instance_type": "t3a.xlarge", From 68e5428d6bfb1378f66c9c418601749f8047f327 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 4 Oct 2023 16:23:59 +0100 Subject: [PATCH 28/31] Force package tests to run when golden images change Signed-off-by: Pedro Algarvio --- tools/ci.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ci.py b/tools/ci.py index 2bf5af5a25f..aaca7e90a3c 100644 --- a/tools/ci.py +++ b/tools/ci.py @@ -370,6 +370,7 @@ def define_jobs( required_pkg_test_changes: set[str] = { changed_files_contents["pkg_tests"], changed_files_contents["workflows"], + changed_files_contents["golden_images"], } if jobs["test-pkg"] and required_pkg_test_changes == {"false"}: if "test:pkg" in labels: From c12bbd199570855e0482a5cb45e8ed270ba35c3b Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 4 Oct 2023 16:42:24 +0100 Subject: [PATCH 29/31] Fix the security groups selection for developer created VMs Signed-off-by: Pedro Algarvio --- tools/vm.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tools/vm.py b/tools/vm.py index 000abc4e40f..9ecc33418c9 100644 --- a/tools/vm.py +++ b/tools/vm.py @@ -984,7 +984,6 @@ class VM: else: # This is a developer running log.info("Starting Developer configured VM") - # Get the develpers security group security_group_filters = [ { "Name": "vpc-id", @@ -994,10 +993,6 @@ class VM: "Name": "tag:spb:project", "Values": ["salt-project"], }, - { - "Name": "tag:spb:developer", - "Values": ["true"], - }, ] response = client.describe_security_groups(Filters=security_group_filters) if not response.get("SecurityGroups"): @@ -1008,6 +1003,26 @@ class VM: self.ctx.exit(1) # Override the launch template network interfaces config security_group_ids = [sg["GroupId"] for sg in response["SecurityGroups"]] + security_group_filters = [ + { + "Name": "vpc-id", + "Values": [vpc.id], + }, + { + "Name": "tag:Name", + "Values": [f"saltproject-{environment}-client-vpn-remote-access"], + }, + ] + response = client.describe_security_groups(Filters=security_group_filters) + if not response.get("SecurityGroups"): + self.ctx.error( + "Could not find the right VPN access security group. " + f"Filters:\n{pprint.pformat(security_group_filters)}" + ) + self.ctx.exit(1) + security_group_ids.extend( + [sg["GroupId"] for sg in response["SecurityGroups"]] + ) progress = create_progress_bar() create_task = progress.add_task( From a7bcc6f90a123fb26175492530fbf6b9f578b09d Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 4 Oct 2023 17:04:03 +0100 Subject: [PATCH 30/31] Log in which environment the developer VM is getting created Signed-off-by: Pedro Algarvio --- tools/vm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/vm.py b/tools/vm.py index 9ecc33418c9..cbb531247d2 100644 --- a/tools/vm.py +++ b/tools/vm.py @@ -983,7 +983,7 @@ class VM: log.info("Starting CI configured VM") else: # This is a developer running - log.info("Starting Developer configured VM") + log.info(f"Starting Developer configured VM In Environment '{environment}'") security_group_filters = [ { "Name": "vpc-id", From a932628c0cc4d417bc63eac19be8a6eacf51e121 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Wed, 4 Oct 2023 15:00:16 +0100 Subject: [PATCH 31/31] Avoid problematic `grub-efi-amd64-signed` package on Ubuntu Signed-off-by: Pedro Algarvio --- pkg/tests/conftest.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/tests/conftest.py b/pkg/tests/conftest.py index 36c60b0e57b..c618b7ed4bb 100644 --- a/pkg/tests/conftest.py +++ b/pkg/tests/conftest.py @@ -49,6 +49,18 @@ def _system_up_to_date( grains, shell, ): + if grains["os"] == "Ubuntu" and grains["osarch"] == "amd64": + # The grub-efi-amd64-signed package seems to be a problem + # right now when updating the system + env = os.environ.copy() + env["DEBIAN_FRONTEND"] = "noninteractive" + ret = shell.run( + "apt-mark", + "hold", + "grub-efi-amd64-signed", + env=env, + ) + assert ret.returncode == 0 if grains["os_family"] == "Debian": ret = shell.run("apt", "update") assert ret.returncode == 0