diff --git a/.coveragerc b/.coveragerc index 2579c96f9cd..40da2c6666d 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,6 +7,11 @@ relative_files = True omit = setup.py .nox/* +source = + pkg + salt + tests + tools [report] # Regexes for lines to exclude from consideration @@ -32,7 +37,7 @@ ignore_errors = True [paths] salt = salt/ - artifacts/salt + artifacts/salt/lib/python3.*/site-packages/salt **/testing/salt/ **\testing\salt tests = diff --git a/.github/workflows/test-action-macos.yml b/.github/workflows/test-action-macos.yml index 18f413ca874..eabbcddac51 100644 --- a/.github/workflows/test-action-macos.yml +++ b/.github/workflows/test-action-macos.yml @@ -415,6 +415,11 @@ jobs: run: | nox -e combine-coverage + - name: Create XML Coverage Reports + if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled' + run: | + nox -e create-xml-coverage-reports + - name: Prepare Test Run Artifacts id: download-artifacts-from-vm if: always() && job.status != 'cancelled' diff --git a/.github/workflows/test-action.yml b/.github/workflows/test-action.yml index 31945d0a8a4..aedb60612a3 100644 --- a/.github/workflows/test-action.yml +++ b/.github/workflows/test-action.yml @@ -373,6 +373,11 @@ jobs: run: | tools --timestamps vm combine-coverage ${{ inputs.distro-slug }} + - name: Create XML Coverage Reports + if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' + run: | + tools --timestamps vm create-xml-coverage-reports ${{ inputs.distro-slug }} + - name: Download Test Run Artifacts id: download-artifacts-from-vm if: always() && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' diff --git a/.github/workflows/test-package-downloads-action-linux.yml b/.github/workflows/test-package-downloads-action-linux.yml index d303aa115df..0d88f596d29 100644 --- a/.github/workflows/test-package-downloads-action-linux.yml +++ b/.github/workflows/test-package-downloads-action-linux.yml @@ -258,6 +258,11 @@ jobs: run: | tools --timestamps vm combine-coverage ${{ inputs.distro-slug }} + - name: Create XML Coverage Reports + if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' + run: | + tools --timestamps vm create-xml-coverage-reports ${{ inputs.distro-slug }} + - name: Download Test Run Artifacts id: download-artifacts-from-vm if: always() && steps.spin-up-vm.outcome == 'success' diff --git a/.github/workflows/test-package-downloads-action-macos.yml b/.github/workflows/test-package-downloads-action-macos.yml index 126aa89c082..2e362959407 100644 --- a/.github/workflows/test-package-downloads-action-macos.yml +++ b/.github/workflows/test-package-downloads-action-macos.yml @@ -242,6 +242,11 @@ jobs: run: | nox --force-color -e combine-coverage + - name: Create XML Coverage Reports + if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled' + run: | + tools --timestamps vm create-xml-coverage-reports ${{ inputs.distro-slug }} + - name: Prepare Test Run Artifacts id: download-artifacts-from-vm if: always() && job.status != 'cancelled' diff --git a/.github/workflows/test-package-downloads-action-windows.yml b/.github/workflows/test-package-downloads-action-windows.yml index 2f9e77fb156..2e20ee8455b 100644 --- a/.github/workflows/test-package-downloads-action-windows.yml +++ b/.github/workflows/test-package-downloads-action-windows.yml @@ -263,6 +263,11 @@ jobs: run: | tools --timestamps vm combine-coverage ${{ inputs.distro-slug }} + - name: Create XML Coverage Reports + if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled' + run: | + tools --timestamps vm create-xml-coverage-reports ${{ inputs.distro-slug }} + - name: Download Test Run Artifacts id: download-artifacts-from-vm if: always() && steps.spin-up-vm.outcome == 'success' diff --git a/noxfile.py b/noxfile.py index ca656f3a9ee..5d20d261da6 100644 --- a/noxfile.py +++ b/noxfile.py @@ -367,9 +367,10 @@ def _install_coverage_requirement(session): ) -def _run_with_coverage(session, *test_cmd, env=None): +def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): _install_coverage_requirement(session) - session.run("coverage", "erase") + if on_rerun is False: + session.run("coverage", "erase") if env is None: env = {} @@ -440,8 +441,8 @@ def _run_with_coverage(session, *test_cmd, env=None): "xml", "-o", str(COVERAGE_OUTPUT_DIR.joinpath("tests.xml").relative_to(REPO_ROOT)), - "--omit=salt/*", - "--include=tests/*", + "--omit=salt/*,artifacts/salt/*", + "--include=tests/*,pkg/tests/*", env=coverage_base_env, ) # Generate report for salt code coverage @@ -450,8 +451,8 @@ def _run_with_coverage(session, *test_cmd, env=None): "xml", "-o", str(COVERAGE_OUTPUT_DIR.joinpath("salt.xml").relative_to(REPO_ROOT)), - "--omit=tests/*", - "--include=salt/*", + "--omit=tests/*,pkg/tests/*", + "--include=salt/*,artifacts/salt/*", env=coverage_base_env, ) # Generate html report for tests code coverage @@ -460,8 +461,8 @@ def _run_with_coverage(session, *test_cmd, env=None): "html", "-d", str(COVERAGE_OUTPUT_DIR.joinpath("html").relative_to(REPO_ROOT)), - "--omit=salt/*", - "--include=tests/*", + "--omit=salt/*,artifacts/salt/*", + "--include=tests/*,pkg/tests/*", env=coverage_base_env, ) # Generate html report for salt code coverage @@ -470,8 +471,8 @@ def _run_with_coverage(session, *test_cmd, env=None): "html", "-d", str(COVERAGE_OUTPUT_DIR.joinpath("html").relative_to(REPO_ROOT)), - "--omit=tests/*", - "--include=salt/*", + "--omit=tests/*,pkg/tests/*", + "--include=salt/*,artifacts/salt/*", env=coverage_base_env, ) @@ -521,8 +522,8 @@ def _report_coverage(session): COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage-salt.json" ) cmd_args = [ - "--omit=tests/*", - "--include=salt/*", + "--omit=tests/*,pkg/tests/*", + "--include=salt/*,artifacts/salt/*", ] elif report_section == "tests": @@ -530,15 +531,15 @@ def _report_coverage(session): COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage-tests.json" ) cmd_args = [ - "--omit=salt/*", - "--include=tests/*", + "--omit=salt/*,artifacts/salt/*", + "--include=tests/*,pkg/tests/*", ] else: json_coverage_file = ( COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage.json" ) cmd_args = [ - "--include=salt/*,tests/*", + "--include=salt/*,artifacts/salt/*,tests/*,pkg/tests/*", ] session.run( @@ -1020,7 +1021,7 @@ def pytest_tornado(session, coverage): session.notify(session_name.replace("pytest-", "test-")) -def _pytest(session, coverage, cmd_args, env=None): +def _pytest(session, coverage, cmd_args, env=None, on_rerun=False): # Create required artifacts directories _create_ci_directories() @@ -1074,6 +1075,7 @@ def _pytest(session, coverage, cmd_args, env=None): "pytest", *args, env=env, + on_rerun=on_rerun, ) else: session.run("python", "-m", "pytest", *args, env=env) @@ -1166,7 +1168,13 @@ def _ci_test(session, transport, onedir=False): ] + chunk_cmd ) - _pytest(session, coverage=track_code_coverage, cmd_args=pytest_args, env=env) + _pytest( + session, + coverage=track_code_coverage, + cmd_args=pytest_args, + env=env, + on_rerun=True, + ) @nox.session(python=_PYTHON_VERSIONS, name="ci-test") @@ -1371,12 +1379,50 @@ def create_html_coverage_report(session): "html", "-d", str(COVERAGE_OUTPUT_DIR.joinpath("html").relative_to(REPO_ROOT)), - "--include=salt/*,tests/*", + "--include=salt/*,artifacts/salt/*,tests/*,pkg/tests/*", "--show-contexts", env=env, ) +@nox.session(python="3", name="create-xml-coverage-reports") +def create_xml_coverage_reports(session): + _install_coverage_requirement(session) + env = { + # The full path to the .coverage data file. Makes sure we always write + # them to the same directory + "COVERAGE_FILE": str(COVERAGE_FILE), + } + + # Generate report for tests code coverage + try: + session.run( + "coverage", + "xml", + "-o", + str(COVERAGE_OUTPUT_DIR.joinpath("tests.xml").relative_to(REPO_ROOT)), + "--omit=salt/*,artifacts/salt/*", + "--include=tests/*,pkg/tests/*", + env=env, + ) + except CommandFailed: + session_warn(session, "Failed to generate the tests XML code coverage report") + + # Generate report for salt code coverage + try: + session.run( + "coverage", + "xml", + "-o", + str(COVERAGE_OUTPUT_DIR.joinpath("salt.xml").relative_to(REPO_ROOT)), + "--omit=tests/*,pkg/tests/*", + "--include=salt/*,artifacts/salt/*", + env=env, + ) + except CommandFailed: + session_warn(session, "Failed to generate the source XML code coverage report") + + class Tee: """ Python class to mimic linux tee behaviour diff --git a/tools/vm.py b/tools/vm.py index 8988fe32f1e..08fcb28b44f 100644 --- a/tools/vm.py +++ b/tools/vm.py @@ -547,6 +547,24 @@ def combine_coverage(ctx: Context, name: str): ctx.exit(returncode) +@vm.command( + name="create-xml-coverage-reports", + arguments={ + "name": { + "help": "The VM Name", + "metavar": "VM_NAME", + }, + }, +) +def create_xml_coverage_reports(ctx: Context, name: str): + """ + Create XML code coverage reports in the VM. + """ + vm = VM(ctx=ctx, name=name, region_name=ctx.parser.options.region) + returncode = vm.create_xml_coverage_reports() + ctx.exit(returncode) + + @vm.command( name="download-artifacts", arguments={ @@ -1415,6 +1433,12 @@ class VM: """ return self.run_nox("combine-coverage", session_args=[self.name]) + def create_xml_coverage_reports(self): + """ + Create XML coverage reports + """ + return self.run_nox("create-xml-coverage-reports", session_args=[self.name]) + def compress_dependencies(self): """ Compress .nox/ into nox..tar.* in the VM