From 94bce224fa46f7b23eb5b80117fdced1f7218e42 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Fri, 9 Jun 2023 15:42:10 +0100 Subject: [PATCH] added xmldiff package for testing expected output vs returned output --- requirements/static/ci/common.in | 1 + requirements/static/ci/py3.10/darwin.txt | 3 ++ requirements/static/ci/py3.10/freebsd.txt | 3 ++ requirements/static/ci/py3.10/linux.txt | 3 ++ requirements/static/ci/py3.10/windows.txt | 3 ++ requirements/static/ci/py3.11/darwin.txt | 3 ++ requirements/static/ci/py3.11/freebsd.txt | 3 ++ requirements/static/ci/py3.11/linux.txt | 3 ++ requirements/static/ci/py3.11/windows.txt | 3 ++ requirements/static/ci/py3.8/freebsd.txt | 3 ++ requirements/static/ci/py3.8/linux.txt | 3 ++ requirements/static/ci/py3.8/windows.txt | 3 ++ requirements/static/ci/py3.9/darwin.txt | 3 ++ requirements/static/ci/py3.9/freebsd.txt | 3 ++ requirements/static/ci/py3.9/linux.txt | 3 ++ requirements/static/ci/py3.9/windows.txt | 3 ++ salt/modules/saltcheck.py | 36 ++++++++++---------- tests/pytests/unit/modules/test_saltcheck.py | 9 +++-- 18 files changed, 71 insertions(+), 20 deletions(-) diff --git a/requirements/static/ci/common.in b/requirements/static/ci/common.in index d2e97267bb3..306e978ac1e 100644 --- a/requirements/static/ci/common.in +++ b/requirements/static/ci/common.in @@ -41,6 +41,7 @@ toml vcert; sys_platform != 'win32' virtualenv>=20.3.0 watchdog>=0.9.0 +xmldiff>=2.4 # Available template libraries that can be used genshi>=0.7.3 cheetah3>=3.2.2 diff --git a/requirements/static/ci/py3.10/darwin.txt b/requirements/static/ci/py3.10/darwin.txt index f5a7941b8ce..213d2dd1110 100644 --- a/requirements/static/ci/py3.10/darwin.txt +++ b/requirements/static/ci/py3.10/darwin.txt @@ -230,6 +230,7 @@ lxml==4.9.2 # junos-eznc # napalm # ncclient + # xmldiff mako==1.2.4 # via -r requirements/static/ci/common.in markupsafe==2.1.2 @@ -558,6 +559,8 @@ werkzeug==2.2.3 # via # moto # pytest-httpserver +xmldiff==2.6.3 + # via -r requirements/static/ci/common.in xmltodict==0.12.0 # via moto yamllint==1.32.0 diff --git a/requirements/static/ci/py3.10/freebsd.txt b/requirements/static/ci/py3.10/freebsd.txt index b08455a8bc5..13e6fa06a4b 100644 --- a/requirements/static/ci/py3.10/freebsd.txt +++ b/requirements/static/ci/py3.10/freebsd.txt @@ -225,6 +225,7 @@ lxml==4.9.2 # junos-eznc # napalm # ncclient + # xmldiff mako==1.2.4 # via -r requirements/static/ci/common.in markupsafe==2.1.2 @@ -547,6 +548,8 @@ werkzeug==2.2.3 # via # moto # pytest-httpserver +xmldiff==2.6.3 + # via -r requirements/static/ci/common.in xmltodict==0.12.0 # via moto yamllint==1.32.0 diff --git a/requirements/static/ci/py3.10/linux.txt b/requirements/static/ci/py3.10/linux.txt index 5b541f900e6..450bfba701d 100644 --- a/requirements/static/ci/py3.10/linux.txt +++ b/requirements/static/ci/py3.10/linux.txt @@ -250,6 +250,7 @@ lxml==4.9.2 # junos-eznc # napalm # ncclient + # xmldiff mako==1.2.4 # via -r requirements/static/ci/common.in markupsafe==2.1.2 @@ -610,6 +611,8 @@ werkzeug==2.2.3 # via # moto # pytest-httpserver +xmldiff==2.6.3 + # via -r requirements/static/ci/common.in xmltodict==0.12.0 # via moto yamllint==1.32.0 diff --git a/requirements/static/ci/py3.10/windows.txt b/requirements/static/ci/py3.10/windows.txt index 39335b19b1f..1c094255a10 100644 --- a/requirements/static/ci/py3.10/windows.txt +++ b/requirements/static/ci/py3.10/windows.txt @@ -203,6 +203,7 @@ lxml==4.9.2 # via # -c requirements/static/ci/../pkg/py3.10/windows.txt # -r requirements/windows.txt + # xmldiff mako==1.2.4 # via -r requirements/static/ci/common.in markupsafe==2.1.2 @@ -494,6 +495,8 @@ wmi==1.5.1 # via # -c requirements/static/ci/../pkg/py3.10/windows.txt # -r requirements/windows.txt +xmldiff==2.6.3 + # via -r requirements/static/ci/common.in xmltodict==0.12.0 # via # moto diff --git a/requirements/static/ci/py3.11/darwin.txt b/requirements/static/ci/py3.11/darwin.txt index aec99646930..923dfef997a 100644 --- a/requirements/static/ci/py3.11/darwin.txt +++ b/requirements/static/ci/py3.11/darwin.txt @@ -228,6 +228,7 @@ lxml==4.9.2 # junos-eznc # napalm # ncclient + # xmldiff mako==1.2.4 # via -r requirements/static/ci/common.in markupsafe==2.1.2 @@ -554,6 +555,8 @@ werkzeug==2.2.3 # via # moto # pytest-httpserver +xmldiff==2.6.3 + # via -r requirements/static/ci/common.in xmltodict==0.12.0 # via moto yamllint==1.32.0 diff --git a/requirements/static/ci/py3.11/freebsd.txt b/requirements/static/ci/py3.11/freebsd.txt index 24dd3991b5a..f96dd5ae430 100644 --- a/requirements/static/ci/py3.11/freebsd.txt +++ b/requirements/static/ci/py3.11/freebsd.txt @@ -223,6 +223,7 @@ lxml==4.9.2 # junos-eznc # napalm # ncclient + # xmldiff mako==1.2.4 # via -r requirements/static/ci/common.in markupsafe==2.1.2 @@ -543,6 +544,8 @@ werkzeug==2.2.3 # via # moto # pytest-httpserver +xmldiff==2.6.3 + # via -r requirements/static/ci/common.in xmltodict==0.12.0 # via moto yamllint==1.32.0 diff --git a/requirements/static/ci/py3.11/linux.txt b/requirements/static/ci/py3.11/linux.txt index 2df8efe5796..cbb2d4a9053 100644 --- a/requirements/static/ci/py3.11/linux.txt +++ b/requirements/static/ci/py3.11/linux.txt @@ -246,6 +246,7 @@ lxml==4.9.2 # junos-eznc # napalm # ncclient + # xmldiff mako==1.2.4 # via -r requirements/static/ci/common.in markupsafe==2.1.2 @@ -604,6 +605,8 @@ werkzeug==2.2.3 # via # moto # pytest-httpserver +xmldiff==2.6.3 + # via -r requirements/static/ci/common.in xmltodict==0.12.0 # via moto yamllint==1.32.0 diff --git a/requirements/static/ci/py3.11/windows.txt b/requirements/static/ci/py3.11/windows.txt index 1985f7f64ad..c03b6b06b90 100644 --- a/requirements/static/ci/py3.11/windows.txt +++ b/requirements/static/ci/py3.11/windows.txt @@ -201,6 +201,7 @@ lxml==4.9.2 # via # -c requirements/static/ci/../pkg/py3.11/windows.txt # -r requirements/windows.txt + # xmldiff mako==1.2.4 # via -r requirements/static/ci/common.in markupsafe==2.1.2 @@ -490,6 +491,8 @@ wmi==1.5.1 # via # -c requirements/static/ci/../pkg/py3.11/windows.txt # -r requirements/windows.txt +xmldiff==2.6.3 + # via -r requirements/static/ci/common.in xmltodict==0.12.0 # via # moto diff --git a/requirements/static/ci/py3.8/freebsd.txt b/requirements/static/ci/py3.8/freebsd.txt index cc878ac14c9..b469c85a988 100644 --- a/requirements/static/ci/py3.8/freebsd.txt +++ b/requirements/static/ci/py3.8/freebsd.txt @@ -229,6 +229,7 @@ lxml==4.9.2 # junos-eznc # napalm # ncclient + # xmldiff mako==1.2.4 # via -r requirements/static/ci/common.in markupsafe==2.1.2 @@ -550,6 +551,8 @@ werkzeug==2.2.3 # via # moto # pytest-httpserver +xmldiff==2.6.3 + # via -r requirements/static/ci/common.in xmltodict==0.12.0 # via moto yamllint==1.32.0 diff --git a/requirements/static/ci/py3.8/linux.txt b/requirements/static/ci/py3.8/linux.txt index 738c5ddc777..85c3518256c 100644 --- a/requirements/static/ci/py3.8/linux.txt +++ b/requirements/static/ci/py3.8/linux.txt @@ -254,6 +254,7 @@ lxml==4.9.2 # junos-eznc # napalm # ncclient + # xmldiff mako==1.2.4 # via -r requirements/static/ci/common.in markupsafe==2.1.2 @@ -614,6 +615,8 @@ werkzeug==2.2.3 # via # moto # pytest-httpserver +xmldiff==2.6.3 + # via -r requirements/static/ci/common.in xmltodict==0.12.0 # via moto yamllint==1.32.0 diff --git a/requirements/static/ci/py3.8/windows.txt b/requirements/static/ci/py3.8/windows.txt index b034b07d0e0..dc76e47cff3 100644 --- a/requirements/static/ci/py3.8/windows.txt +++ b/requirements/static/ci/py3.8/windows.txt @@ -209,6 +209,7 @@ lxml==4.9.2 # via # -c requirements/static/ci/../pkg/py3.8/windows.txt # -r requirements/windows.txt + # xmldiff mako==1.2.4 # via -r requirements/static/ci/common.in markupsafe==2.1.2 @@ -502,6 +503,8 @@ wmi==1.5.1 # via # -c requirements/static/ci/../pkg/py3.8/windows.txt # -r requirements/windows.txt +xmldiff==2.6.3 + # via -r requirements/static/ci/common.in xmltodict==0.12.0 # via # moto diff --git a/requirements/static/ci/py3.9/darwin.txt b/requirements/static/ci/py3.9/darwin.txt index 52fdef985ba..47b171d48c5 100644 --- a/requirements/static/ci/py3.9/darwin.txt +++ b/requirements/static/ci/py3.9/darwin.txt @@ -230,6 +230,7 @@ lxml==4.9.2 # junos-eznc # napalm # ncclient + # xmldiff mako==1.2.4 # via -r requirements/static/ci/common.in markupsafe==2.1.2 @@ -558,6 +559,8 @@ werkzeug==2.2.3 # via # moto # pytest-httpserver +xmldiff==2.6.3 + # via -r requirements/static/ci/common.in xmltodict==0.12.0 # via moto yamllint==1.32.0 diff --git a/requirements/static/ci/py3.9/freebsd.txt b/requirements/static/ci/py3.9/freebsd.txt index 51c685f7551..397992d6b4c 100644 --- a/requirements/static/ci/py3.9/freebsd.txt +++ b/requirements/static/ci/py3.9/freebsd.txt @@ -225,6 +225,7 @@ lxml==4.9.2 # junos-eznc # napalm # ncclient + # xmldiff mako==1.2.4 # via -r requirements/static/ci/common.in markupsafe==2.1.2 @@ -547,6 +548,8 @@ werkzeug==2.2.3 # via # moto # pytest-httpserver +xmldiff==2.6.3 + # via -r requirements/static/ci/common.in xmltodict==0.12.0 # via moto yamllint==1.32.0 diff --git a/requirements/static/ci/py3.9/linux.txt b/requirements/static/ci/py3.9/linux.txt index 739ea7880e2..1df8cbef8ef 100644 --- a/requirements/static/ci/py3.9/linux.txt +++ b/requirements/static/ci/py3.9/linux.txt @@ -252,6 +252,7 @@ lxml==4.9.2 # junos-eznc # napalm # ncclient + # xmldiff mako==1.2.4 # via -r requirements/static/ci/common.in markupsafe==2.1.2 @@ -612,6 +613,8 @@ werkzeug==2.2.3 # via # moto # pytest-httpserver +xmldiff==2.6.3 + # via -r requirements/static/ci/common.in xmltodict==0.12.0 # via moto yamllint==1.32.0 diff --git a/requirements/static/ci/py3.9/windows.txt b/requirements/static/ci/py3.9/windows.txt index 99b2570b61c..77410538ef2 100644 --- a/requirements/static/ci/py3.9/windows.txt +++ b/requirements/static/ci/py3.9/windows.txt @@ -205,6 +205,7 @@ lxml==4.9.2 # via # -c requirements/static/ci/../pkg/py3.9/windows.txt # -r requirements/windows.txt + # xmldiff mako==1.2.4 # via -r requirements/static/ci/common.in markupsafe==2.1.2 @@ -498,6 +499,8 @@ wmi==1.5.1 # via # -c requirements/static/ci/../pkg/py3.9/windows.txt # -r requirements/windows.txt +xmldiff==2.6.3 + # via -r requirements/static/ci/common.in xmltodict==0.12.0 # via # moto diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py index 222f4eb7eec..9355fdfa76b 100644 --- a/salt/modules/saltcheck.py +++ b/salt/modules/saltcheck.py @@ -646,7 +646,7 @@ def _generate_junit_out_list(results): if not results[state]: test_cases.append(TestCase("missing_test", "", "", "Test(s) Missing")) else: - for name, val in results[state].items(): + for name, val in sorted(results[state].items()): time = float(val["duration"]) status = val["status"] test_cases.append(TestCase(name, "", round(time, 4))) @@ -656,12 +656,12 @@ def _generate_junit_out_list(results): if status.startswith("Skip"): test_cases[len(test_cases) - 1].add_skipped_info(status) total_time = total_time + float(val["duration"]) - test_suite = TestSuite("test_results", test_cases) # Set exist code to 1 if failed tests # Use-cases for exist code handling of missing or skipped? __context__["retcode"] = failed - return TestSuite.to_xml_string([test_suite]) + xml_string = TestSuite.to_xml_string([test_suite]) + return xml_string def _render_file(file_path): @@ -694,7 +694,7 @@ def _is_valid_function(module_name, function): functions = __salt__["sys.list_functions"](module_name) except salt.exceptions.SaltException: functions = ["unable to look up functions"] - return "{}.{}".format(module_name, function) in functions + return f"{module_name}.{function}" in functions def _get_top_states(saltenv="base"): @@ -921,19 +921,19 @@ class SaltCheck: if output_details: if assertion_section: assertion_section_repr_title = " {}".format("assertion_section") - assertion_section_repr_value = " {}".format(assertion_section) + assertion_section_repr_value = f" {assertion_section}" else: assertion_section_repr_title = "" assertion_section_repr_value = "" value[ - "module.function [args]{}".format(assertion_section_repr_title) + f"module.function [args]{assertion_section_repr_title}" ] = "{} {}{}".format( mod_and_func, dumps(args), assertion_section_repr_value, ) value["saltcheck assertion"] = "{}{} {}".format( - ("" if expected_return is None else "{} ".format(expected_return)), + ("" if expected_return is None else f"{expected_return} "), assertion_desc, ("hidden" if not assert_print_result else module_output), ) @@ -978,7 +978,7 @@ class SaltCheck: for num, assert_group in enumerate( test_dict.get("assertions"), start=1 ): - result["assertion{}".format(num)] = self._run_assertions( + result[f"assertion{num}"] = self._run_assertions( mod_and_func, args, assert_group, @@ -1078,7 +1078,7 @@ class SaltCheck: """ result = "Pass" try: - assert returned is True, "{} not True".format(returned) + assert returned is True, f"{returned} not True" except AssertionError as err: result = "Fail: " + str(err) return result @@ -1092,7 +1092,7 @@ class SaltCheck: if isinstance(returned, str): returned = bool(returned) try: - assert returned is False, "{} not False".format(returned) + assert returned is False, f"{returned} not False" except AssertionError as err: result = "Fail: " + str(err) return result @@ -1138,7 +1138,7 @@ class SaltCheck: """ result = "Pass" try: - assert expected > returned, "{} not False".format(returned) + assert expected > returned, f"{returned} not False" except AssertionError as err: result = "Fail: " + str(err) return result @@ -1150,7 +1150,7 @@ class SaltCheck: """ result = "Pass" try: - assert expected >= returned, "{} not False".format(returned) + assert expected >= returned, f"{returned} not False" except AssertionError as err: result = "Fail: " + str(err) return result @@ -1162,7 +1162,7 @@ class SaltCheck: """ result = "Pass" try: - assert expected < returned, "{} not False".format(returned) + assert expected < returned, f"{returned} not False" except AssertionError as err: result = "Fail: " + str(err) return result @@ -1174,7 +1174,7 @@ class SaltCheck: """ result = "Pass" try: - assert expected <= returned, "{} not False".format(returned) + assert expected <= returned, f"{returned} not False" except AssertionError as err: result = "Fail: " + str(err) return result @@ -1186,7 +1186,7 @@ class SaltCheck: """ result = "Pass" try: - assert not returned, "{} is not empty".format(returned) + assert not returned, f"{returned} is not empty" except AssertionError as err: result = "Fail: " + str(err) return result @@ -1298,7 +1298,7 @@ class StateTestLoader: all_sls_paths.append(test_path) state_name_base = state_name.split(".")[0] - test_path = "salt://{}/{}".format(state_name_base, self.saltcheck_test_location) + test_path = f"salt://{state_name_base}/{self.saltcheck_test_location}" all_sls_paths.append(test_path) unique_paths = set(all_sls_paths) @@ -1414,13 +1414,13 @@ class StateTestLoader: os.path.join( os.sep.join(split_sls[: len(split_sls) - 1]), os.path.normpath(self.saltcheck_test_location), - "{}.tst".format(split_sls[-1]), + f"{split_sls[-1]}.tst", ), os.path.join( split_sls[0], os.path.normpath(self.saltcheck_test_location), os.sep.join(split_sls[1:-1]), - "{}.tst".format(split_sls[-1]), + f"{split_sls[-1]}.tst", ), } # for this state, find matching test files and load them diff --git a/tests/pytests/unit/modules/test_saltcheck.py b/tests/pytests/unit/modules/test_saltcheck.py index b8574cea8cb..8b7ff55213f 100644 --- a/tests/pytests/unit/modules/test_saltcheck.py +++ b/tests/pytests/unit/modules/test_saltcheck.py @@ -3,6 +3,11 @@ import pytest import salt.modules.saltcheck as saltcheck from tests.support.mock import MagicMock +try: + from xmldiff import main +except ImportError: + raise ImportError + @pytest.fixture() def configure_loader_modules(): @@ -36,6 +41,6 @@ def test__generate_junit_out_list(): + """\t\t\n\t\t\t\n\t\t\n""" + """\t\n\n""" ) - ret = saltcheck._generate_junit_out_list(results) - assert ret == expected + diff = main.diff_texts(ret, expected) + assert diff == []