From 2430a0eeb3e042bc2c14059b62fb7631388a0d26 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 17 Sep 2023 18:06:08 +0100 Subject: [PATCH 1/3] Don't erase collected coverage data on test failure re-runs Fixes #65214 Signed-off-by: Pedro Algarvio --- noxfile.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/noxfile.py b/noxfile.py index 6f5e9a9d7df..e963b98427b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -369,9 +369,10 @@ def _install_coverage_requirement(session): ) -def _run_with_coverage(session, *test_cmd, env=None): +def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): _install_coverage_requirement(session) - session.run("coverage", "erase") + if on_rerun is False: + session.run("coverage", "erase") if env is None: env = {} @@ -1013,7 +1014,7 @@ def pytest_tornado(session, coverage): session.notify(session_name.replace("pytest-", "test-")) -def _pytest(session, coverage, cmd_args, env=None): +def _pytest(session, coverage, cmd_args, env=None, on_rerun=False): # Create required artifacts directories _create_ci_directories() @@ -1067,6 +1068,7 @@ def _pytest(session, coverage, cmd_args, env=None): "pytest", *args, env=env, + on_rerun=on_rerun, ) else: session.run("python", "-m", "pytest", *args, env=env) @@ -1159,7 +1161,13 @@ def _ci_test(session, transport, onedir=False): ] + chunk_cmd ) - _pytest(session, coverage=track_code_coverage, cmd_args=pytest_args, env=env) + _pytest( + session, + coverage=track_code_coverage, + cmd_args=pytest_args, + env=env, + on_rerun=True, + ) @nox.session(python=_PYTHON_VERSIONS, name="ci-test") From 29a57d2d3276e0e3e9b44b96a1153ddc4656b335 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 17 Sep 2023 12:45:01 +0100 Subject: [PATCH 2/3] Generate the XML code coverage reports before trying to use them Signed-off-by: Pedro Algarvio --- .github/workflows/test-action-macos.yml | 5 +++ .github/workflows/test-action.yml | 5 +++ .../test-package-downloads-action-linux.yml | 5 +++ .../test-package-downloads-action-macos.yml | 5 +++ .../test-package-downloads-action-windows.yml | 5 +++ noxfile.py | 38 +++++++++++++++++++ tools/vm.py | 24 ++++++++++++ 7 files changed, 87 insertions(+) diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index 18f413ca874..eabbcddac51 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -415,6 +415,11 @@ jobs: run: | nox -e combine-coverage + - name: Create XML Coverage Reports + if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled' + run: | + nox -e create-xml-coverage-reports + - name: Prepare Test Run Artifacts id: download-artifacts-from-vm if: always() && job.status != 'cancelled' diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index 31945d0a8a4..aedb60612a3 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -373,6 +373,11 @@ jobs: run: | tools --timestamps vm combine-coverage ${{ inputs.distro-slug }} + - name: Create XML Coverage Reports + if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' + run: | + tools --timestamps vm create-xml-coverage-reports ${{ inputs.distro-slug }} + - name: Download Test Run Artifacts id: download-artifacts-from-vm if: always() && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' diff --git a/.github/workflows/test-package-downloads-action-linux.yml b/.github/workflows/test-package-downloads-action-linux.yml index d303aa115df..0d88f596d29 100644 --- a/.github/workflows/test-package-downloads-action-linux.yml +++ b/.github/workflows/test-package-downloads-action-linux.yml @@ -258,6 +258,11 @@ jobs: run: | tools --timestamps vm combine-coverage ${{ inputs.distro-slug }} + - name: Create XML Coverage Reports + if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' + run: | + tools --timestamps vm create-xml-coverage-reports ${{ inputs.distro-slug }} + - name: Download Test Run Artifacts id: download-artifacts-from-vm if: always() && steps.spin-up-vm.outcome == 'success' diff --git a/.github/workflows/test-package-downloads-action-macos.yml b/.github/workflows/test-package-downloads-action-macos.yml index 126aa89c082..2e362959407 100644 --- a/.github/workflows/test-package-downloads-action-macos.yml +++ b/.github/workflows/test-package-downloads-action-macos.yml @@ -242,6 +242,11 @@ jobs: run: | nox --force-color -e combine-coverage + - name: Create XML Coverage Reports + if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled' + run: | + tools --timestamps vm create-xml-coverage-reports ${{ inputs.distro-slug }} + - name: Prepare Test Run Artifacts id: download-artifacts-from-vm if: always() && job.status != 'cancelled' diff --git a/.github/workflows/test-package-downloads-action-windows.yml b/.github/workflows/test-package-downloads-action-windows.yml index 2f9e77fb156..2e20ee8455b 100644 --- a/.github/workflows/test-package-downloads-action-windows.yml +++ b/.github/workflows/test-package-downloads-action-windows.yml @@ -263,6 +263,11 @@ jobs: run: | tools --timestamps vm combine-coverage ${{ inputs.distro-slug }} + - name: Create XML Coverage Reports + if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' + run: | + tools --timestamps vm create-xml-coverage-reports ${{ inputs.distro-slug }} + - name: Download Test Run Artifacts id: download-artifacts-from-vm if: always() && steps.spin-up-vm.outcome == 'success' diff --git a/noxfile.py b/noxfile.py index e963b98427b..0c634474e81 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1378,6 +1378,44 @@ def create_html_coverage_report(session): ) +@nox.session(python="3", name="create-xml-coverage-reports") +def create_xml_coverage_reports(session): + _install_coverage_requirement(session) + env = { + # The full path to the .coverage data file. Makes sure we always write + # them to the same directory + "COVERAGE_FILE": str(COVERAGE_FILE), + } + + # Generate report for tests code coverage + try: + session.run( + "coverage", + "xml", + "-o", + str(COVERAGE_OUTPUT_DIR.joinpath("tests.xml").relative_to(REPO_ROOT)), + "--omit=salt/*,artifacts/salt/*", + "--include=tests/*", + env=env, + ) + except CommandFailed: + session_warn(session, "Failed to generate the tests XML code coverage report") + + # Generate report for salt code coverage + try: + session.run( + "coverage", + "xml", + "-o", + str(COVERAGE_OUTPUT_DIR.joinpath("salt.xml").relative_to(REPO_ROOT)), + "--omit=tests/*", + "--include=salt/*,artifacts/salt/*", + env=env, + ) + except CommandFailed: + session_warn(session, "Failed to generate the source XML code coverage report") + + class Tee: """ Python class to mimic linux tee behaviour diff --git a/tools/vm.py b/tools/vm.py index 8988fe32f1e..08fcb28b44f 100644 --- a/tools/vm.py +++ b/tools/vm.py @@ -547,6 +547,24 @@ def combine_coverage(ctx: Context, name: str): ctx.exit(returncode) +@vm.command( + name="create-xml-coverage-reports", + arguments={ + "name": { + "help": "The VM Name", + "metavar": "VM_NAME", + }, + }, +) +def create_xml_coverage_reports(ctx: Context, name: str): + """ + Create XML code coverage reports in the VM. + """ + vm = VM(ctx=ctx, name=name, region_name=ctx.parser.options.region) + returncode = vm.create_xml_coverage_reports() + ctx.exit(returncode) + + @vm.command( name="download-artifacts", arguments={ @@ -1415,6 +1433,12 @@ class VM: """ return self.run_nox("combine-coverage", session_args=[self.name]) + def create_xml_coverage_reports(self): + """ + Create XML coverage reports + """ + return self.run_nox("create-xml-coverage-reports", session_args=[self.name]) + def compress_dependencies(self): """ Compress .nox/ into nox..tar.* in the VM From e8115ec126f51f83c99fec3a5f30bf4067c9ac93 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 18 Sep 2023 14:44:04 +0100 Subject: [PATCH 3/3] Fix `onedir` code coverage collection and reporting Signed-off-by: Pedro Algarvio --- .coveragerc | 7 ++++++- noxfile.py | 32 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/.coveragerc b/.coveragerc index 2579c96f9cd..40da2c6666d 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,6 +7,11 @@ relative_files = True omit = setup.py .nox/* +source = + pkg + salt + tests + tools [report] # Regexes for lines to exclude from consideration @@ -32,7 +37,7 @@ ignore_errors = True [paths] salt = salt/ - artifacts/salt + artifacts/salt/lib/python3.*/site-packages/salt **/testing/salt/ **\testing\salt tests = diff --git a/noxfile.py b/noxfile.py index 0c634474e81..586b3231418 100644 --- a/noxfile.py +++ b/noxfile.py @@ -443,8 +443,8 @@ def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): "xml", "-o", str(COVERAGE_OUTPUT_DIR.joinpath("tests.xml").relative_to(REPO_ROOT)), - "--omit=salt/*", - "--include=tests/*", + "--omit=salt/*,artifacts/salt/*", + "--include=tests/*,pkg/tests/*", env=coverage_base_env, ) # Generate report for salt code coverage @@ -453,8 +453,8 @@ def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): "xml", "-o", str(COVERAGE_OUTPUT_DIR.joinpath("salt.xml").relative_to(REPO_ROOT)), - "--omit=tests/*", - "--include=salt/*", + "--omit=tests/*,pkg/tests/*", + "--include=salt/*,artifacts/salt/*", env=coverage_base_env, ) # Generate html report for tests code coverage @@ -463,8 +463,8 @@ def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): "html", "-d", str(COVERAGE_OUTPUT_DIR.joinpath("html").relative_to(REPO_ROOT)), - "--omit=salt/*", - "--include=tests/*", + "--omit=salt/*,artifacts/salt/*", + "--include=tests/*,pkg/tests/*", env=coverage_base_env, ) # Generate html report for salt code coverage @@ -473,8 +473,8 @@ def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): "html", "-d", str(COVERAGE_OUTPUT_DIR.joinpath("html").relative_to(REPO_ROOT)), - "--omit=tests/*", - "--include=salt/*", + "--omit=tests/*,pkg/tests/*", + "--include=salt/*,artifacts/salt/*", env=coverage_base_env, ) @@ -524,8 +524,8 @@ def _report_coverage(session): COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage-salt.json" ) cmd_args = [ - "--omit=tests/*", - "--include=salt/*", + "--omit=tests/*,pkg/tests/*", + "--include=salt/*,artifacts/salt/*", ] elif report_section == "tests": @@ -533,15 +533,15 @@ def _report_coverage(session): COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage-tests.json" ) cmd_args = [ - "--omit=salt/*", - "--include=tests/*", + "--omit=salt/*,artifacts/salt/*", + "--include=tests/*,pkg/tests/*", ] else: json_coverage_file = ( COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage.json" ) cmd_args = [ - "--include=salt/*,tests/*", + "--include=salt/*,artifacts/salt/*,tests/*,pkg/tests/*", ] session.run( @@ -1372,7 +1372,7 @@ def create_html_coverage_report(session): "html", "-d", str(COVERAGE_OUTPUT_DIR.joinpath("html").relative_to(REPO_ROOT)), - "--include=salt/*,tests/*", + "--include=salt/*,artifacts/salt/*,tests/*,pkg/tests/*", "--show-contexts", env=env, ) @@ -1395,7 +1395,7 @@ def create_xml_coverage_reports(session): "-o", str(COVERAGE_OUTPUT_DIR.joinpath("tests.xml").relative_to(REPO_ROOT)), "--omit=salt/*,artifacts/salt/*", - "--include=tests/*", + "--include=tests/*,pkg/tests/*", env=env, ) except CommandFailed: @@ -1408,7 +1408,7 @@ def create_xml_coverage_reports(session): "xml", "-o", str(COVERAGE_OUTPUT_DIR.joinpath("salt.xml").relative_to(REPO_ROOT)), - "--omit=tests/*", + "--omit=tests/*,pkg/tests/*", "--include=salt/*,artifacts/salt/*", env=env, )