From ef30291e53255e132c174d44d0d27c1789998874 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 2 Oct 2023 19:44:40 +0100 Subject: [PATCH] Fix code coverage collection/reporting. Create Salt specific coverage report. Signed-off-by: Pedro Algarvio --- .coveragerc | 6 +- .github/workflows/ci.yml | 20 ++++- .github/workflows/nightly.yml | 20 ++++- .github/workflows/scheduled.yml | 20 ++++- .github/workflows/templates/ci.yml.jinja | 20 ++++- noxfile.py | 96 +++++++++++++++++++----- 6 files changed, 146 insertions(+), 36 deletions(-) diff --git a/.coveragerc b/.coveragerc index 40da2c6666d..bdd2587d114 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,11 +7,13 @@ relative_files = True omit = setup.py .nox/* -source = - pkg +source_pkgs = + pkg.tests salt tests tools +disable_warnings = module-not-imported + [report] # Regexes for lines to exclude from consideration diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8165761d85c..b99a0a6ac77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2140,15 +2140,27 @@ jobs: run: | nox --force-color -e combine-coverage - - name: Create Code Coverage HTML Report + - name: Create Salt Code Coverage HTML Report + run: | + nox --force-color -e create-html-coverage-report -- salt + + - name: Upload Salt Code Coverage HTML Report + uses: actions/upload-artifact@v3 + with: + name: code-coverage-salt-html-report + path: artifacts/coverage/html/salt + retention-days: 7 + if-no-files-found: error + + - name: Create Full Code Coverage HTML Report run: | nox --force-color -e create-html-coverage-report - - name: Upload Code Coverage HTML Report + - name: Upload Full Code Coverage HTML Report uses: actions/upload-artifact@v3 with: - name: code-coverage-html-report - path: artifacts/coverage/html + name: code-coverage-full-html-report + path: artifacts/coverage/html/full retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9475cccff58..70dd302031a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -2201,15 +2201,27 @@ jobs: run: | nox --force-color -e combine-coverage - - name: Create Code Coverage HTML Report + - name: Create Salt Code Coverage HTML Report + run: | + nox --force-color -e create-html-coverage-report -- salt + + - name: Upload Salt Code Coverage HTML Report + uses: actions/upload-artifact@v3 + with: + name: code-coverage-salt-html-report + path: artifacts/coverage/html/salt + retention-days: 7 + if-no-files-found: error + + - name: Create Full Code Coverage HTML Report run: | nox --force-color -e create-html-coverage-report - - name: Upload Code Coverage HTML Report + - name: Upload Full Code Coverage HTML Report uses: actions/upload-artifact@v3 with: - name: code-coverage-html-report - path: artifacts/coverage/html + name: code-coverage-full-html-report + path: artifacts/coverage/html/full retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 6ac5e68bc52..f8bb593c84c 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -2174,15 +2174,27 @@ jobs: run: | nox --force-color -e combine-coverage - - name: Create Code Coverage HTML Report + - name: Create Salt Code Coverage HTML Report + run: | + nox --force-color -e create-html-coverage-report -- salt + + - name: Upload Salt Code Coverage HTML Report + uses: actions/upload-artifact@v3 + with: + name: code-coverage-salt-html-report + path: artifacts/coverage/html/salt + retention-days: 7 + if-no-files-found: error + + - name: Create Full Code Coverage HTML Report run: | nox --force-color -e create-html-coverage-report - - name: Upload Code Coverage HTML Report + - name: Upload Full Code Coverage HTML Report uses: actions/upload-artifact@v3 with: - name: code-coverage-html-report - path: artifacts/coverage/html + name: code-coverage-full-html-report + path: artifacts/coverage/html/full retention-days: 7 if-no-files-found: error diff --git a/.github/workflows/templates/ci.yml.jinja b/.github/workflows/templates/ci.yml.jinja index 67d8d1f877b..e5d0f6171b8 100644 --- a/.github/workflows/templates/ci.yml.jinja +++ b/.github/workflows/templates/ci.yml.jinja @@ -355,15 +355,27 @@ run: | nox --force-color -e combine-coverage - - name: Create Code Coverage HTML Report + - name: Create Salt Code Coverage HTML Report + run: | + nox --force-color -e create-html-coverage-report -- salt + + - name: Upload Salt Code Coverage HTML Report + uses: actions/upload-artifact@v3 + with: + name: code-coverage-salt-html-report + path: artifacts/coverage/html/salt + retention-days: 7 + if-no-files-found: error + + - name: Create Full Code Coverage HTML Report run: | nox --force-color -e create-html-coverage-report - - name: Upload Code Coverage HTML Report + - name: Upload Full Code Coverage HTML Report uses: actions/upload-artifact@v3 with: - name: code-coverage-html-report - path: artifacts/coverage/html + name: code-coverage-full-html-report + path: artifacts/coverage/html/full retention-days: 7 if-no-files-found: error <%- endif %> diff --git a/noxfile.py b/noxfile.py index 313b31a2c18..7ebca469e84 100644 --- a/noxfile.py +++ b/noxfile.py @@ -405,7 +405,9 @@ def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): # Always combine and generate the XML coverage report try: session.run( - "coverage", "combine", "--debug=pathmap", env=coverage_base_env + "coverage", + "combine", + env=coverage_base_env, ) except CommandFailed: # Sometimes some of the coverage files are corrupt which would trigger a CommandFailed @@ -417,7 +419,7 @@ def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): "xml", "-o", str(COVERAGE_OUTPUT_DIR.joinpath("tests.xml").relative_to(REPO_ROOT)), - "--omit=salt/*,artifacts/salt/*", + "--omit=salt/*", "--include=tests/*,pkg/tests/*", env=coverage_base_env, ) @@ -428,7 +430,7 @@ def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): "-o", str(COVERAGE_OUTPUT_DIR.joinpath("salt.xml").relative_to(REPO_ROOT)), "--omit=tests/*,pkg/tests/*", - "--include=salt/*,artifacts/salt/*", + "--include=salt/*", env=coverage_base_env, ) # Generate html report for tests code coverage @@ -437,7 +439,7 @@ def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): "html", "-d", str(COVERAGE_OUTPUT_DIR.joinpath("html").relative_to(REPO_ROOT)), - "--omit=salt/*,artifacts/salt/*", + "--omit=salt/*", "--include=tests/*,pkg/tests/*", env=coverage_base_env, ) @@ -448,7 +450,7 @@ def _run_with_coverage(session, *test_cmd, env=None, on_rerun=False): "-d", str(COVERAGE_OUTPUT_DIR.joinpath("html").relative_to(REPO_ROOT)), "--omit=tests/*,pkg/tests/*", - "--include=salt/*,artifacts/salt/*", + "--include=salt/*", env=coverage_base_env, ) @@ -499,7 +501,7 @@ def _report_coverage(session): ) cmd_args = [ "--omit=tests/*,pkg/tests/*", - "--include=salt/*,artifacts/salt/*", + "--include=salt/*", ] elif report_section == "tests": @@ -507,7 +509,7 @@ def _report_coverage(session): COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage-tests.json" ) cmd_args = [ - "--omit=salt/*,artifacts/salt/*", + "--omit=salt/*", "--include=tests/*,pkg/tests/*", ] else: @@ -515,9 +517,16 @@ def _report_coverage(session): COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage.json" ) cmd_args = [ - "--include=salt/*,artifacts/salt/*,tests/*,pkg/tests/*", + "--include=salt/*,tests/*,pkg/tests/*", ] + session.run( + "coverage", + "report", + *cmd_args, + env=env, + ) + session.run( "coverage", "json", @@ -526,12 +535,6 @@ def _report_coverage(session): *cmd_args, env=env, ) - session.run( - "coverage", - "report", - *cmd_args, - env=env, - ) @nox.session(python=_PYTHON_VERSIONS, name="test-parametrized") @@ -1379,14 +1382,71 @@ def create_html_coverage_report(session): "COVERAGE_FILE": str(COVERAGE_FILE), } + report_section = None + if session.posargs: + report_section = session.posargs.pop(0) + if report_section not in ("salt", "tests"): + session.error("The report section can only be one of 'salt', 'tests'.") + if session.posargs: + session.error( + "Only one argument can be passed to the session, which is optional " + "and is one of 'salt', 'tests'." + ) + + if not IS_WINDOWS: + # The coverage file might have come from a windows machine, fix paths + with sqlite3.connect(COVERAGE_FILE) as db: + res = db.execute(r"SELECT * FROM file WHERE path LIKE '%salt\%'") + if res.fetchone(): + session_warn( + session, + "Replacing backwards slashes with forward slashes on file " + "paths in the coverage database", + ) + db.execute(r"UPDATE OR IGNORE file SET path=replace(path, '\', '/');") + + if report_section == "salt": + report_dir = str( + COVERAGE_OUTPUT_DIR.joinpath("html", "salt").relative_to(REPO_ROOT) + ) + json_coverage_file = ( + COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage-salt.json" + ) + cmd_args = [ + "--omit=tests/*,pkg/tests/*", + "--include=salt/*", + ] + + elif report_section == "tests": + report_dir = str( + COVERAGE_OUTPUT_DIR.joinpath("html", "tests").relative_to(REPO_ROOT) + ) + json_coverage_file = ( + COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage-tests.json" + ) + cmd_args = [ + "--omit=salt/*", + "--include=tests/*,pkg/tests/*", + ] + else: + report_dir = str( + COVERAGE_OUTPUT_DIR.joinpath("html", "full").relative_to(REPO_ROOT) + ) + json_coverage_file = ( + COVERAGE_OUTPUT_DIR.relative_to(REPO_ROOT) / "coverage.json" + ) + cmd_args = [ + "--include=salt/*,tests/*,pkg/tests/*", + ] + # Generate html report for Salt and tests combined code coverage session.run( "coverage", "html", "-d", - str(COVERAGE_OUTPUT_DIR.joinpath("html").relative_to(REPO_ROOT)), - "--include=salt/*,artifacts/salt/*,tests/*,pkg/tests/*", + report_dir, "--show-contexts", + *cmd_args, env=env, ) @@ -1406,7 +1466,7 @@ def _create_xml_coverage_reports(session): "xml", "-o", str(COVERAGE_OUTPUT_DIR.joinpath("tests.xml").relative_to(REPO_ROOT)), - "--omit=salt/*,artifacts/salt/*", + "--omit=salt/*", "--include=tests/*,pkg/tests/*", env=env, ) @@ -1421,7 +1481,7 @@ def _create_xml_coverage_reports(session): "-o", str(COVERAGE_OUTPUT_DIR.joinpath("salt.xml").relative_to(REPO_ROOT)), "--omit=tests/*,pkg/tests/*", - "--include=salt/*,artifacts/salt/*", + "--include=salt/*", env=env, ) except CommandFailed: