From e6e1ab3ef698eaeeaec84499f7b6b1c1873b6668 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Tue, 19 May 2020 09:47:45 +0100 Subject: [PATCH] Add task to check `tests/filename_map.yml` validity --- .pre-commit-config.yaml | 14 +++++ requirements/static/invoke.in | 1 + requirements/static/py3.5/invoke.txt | 1 + requirements/static/py3.6/invoke.txt | 1 + requirements/static/py3.7/invoke.txt | 1 + requirements/static/py3.8/invoke.txt | 1 + requirements/static/py3.9/invoke.txt | 1 + tasks/__init__.py | 3 +- tasks/filemap.py | 94 ++++++++++++++++++++++++++++ 9 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 tasks/filemap.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5da99877107..5dcb9609c78 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -638,3 +638,17 @@ repos: - invoke-pre-commit - -- - docs.check + + - repo: https://github.com/saltstack/salt-nox-pre-commit + rev: master + hooks: + - id: nox-py2 + alias: check-filemap + name: Check Filename Map Change Matching + files: ^tests/filename_map\.yml$ + pass_filenames: false + args: + - -e + - invoke-pre-commit + - -- + - filemap.check diff --git a/requirements/static/invoke.in b/requirements/static/invoke.in index 2c48557fd18..87633def656 100644 --- a/requirements/static/invoke.in +++ b/requirements/static/invoke.in @@ -1,2 +1,3 @@ invoke blessings +pyyaml diff --git a/requirements/static/py3.5/invoke.txt b/requirements/static/py3.5/invoke.txt index 3386dce93ae..f55b1e13a9e 100644 --- a/requirements/static/py3.5/invoke.txt +++ b/requirements/static/py3.5/invoke.txt @@ -6,4 +6,5 @@ # blessings==1.7 invoke==1.4.1 +pyyaml==5.3.1 six==1.14.0 # via blessings diff --git a/requirements/static/py3.6/invoke.txt b/requirements/static/py3.6/invoke.txt index 0272e488434..22467b1d345 100644 --- a/requirements/static/py3.6/invoke.txt +++ b/requirements/static/py3.6/invoke.txt @@ -6,4 +6,5 @@ # blessings==1.7 invoke==1.4.1 +pyyaml==5.3.1 six==1.14.0 # via blessings diff --git a/requirements/static/py3.7/invoke.txt b/requirements/static/py3.7/invoke.txt index b2784255ab4..cc36ccd9143 100644 --- a/requirements/static/py3.7/invoke.txt +++ b/requirements/static/py3.7/invoke.txt @@ -6,4 +6,5 @@ # blessings==1.7 invoke==1.4.1 +pyyaml==5.3.1 six==1.14.0 # via blessings diff --git a/requirements/static/py3.8/invoke.txt b/requirements/static/py3.8/invoke.txt index 4f1cd2b1c0c..4ca4934147f 100644 --- a/requirements/static/py3.8/invoke.txt +++ b/requirements/static/py3.8/invoke.txt @@ -6,4 +6,5 @@ # blessings==1.7 invoke==1.4.1 +pyyaml==5.3.1 six==1.14.0 # via blessings diff --git a/requirements/static/py3.9/invoke.txt b/requirements/static/py3.9/invoke.txt index 9e101a5d86c..be86f41c053 100644 --- a/requirements/static/py3.9/invoke.txt +++ b/requirements/static/py3.9/invoke.txt @@ -6,4 +6,5 @@ # blessings==1.7 invoke==1.4.1 +pyyaml==5.3.1 six==1.14.0 # via blessings diff --git a/tasks/__init__.py b/tasks/__init__.py index e0ec37858aa..7c9518530c5 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -2,8 +2,9 @@ from invoke import Collection # pylint: disable=3rd-party-module-not-gated -from . import docs, loader +from . import docs, filemap, loader ns = Collection() ns.add_collection(Collection.from_module(docs, name="docs"), name="docs") ns.add_collection(Collection.from_module(loader, name="loader"), name="loader") +ns.add_collection(Collection.from_module(filemap, name="filemap"), name="filemap") diff --git a/tasks/filemap.py b/tasks/filemap.py new file mode 100644 index 00000000000..42d69879cce --- /dev/null +++ b/tasks/filemap.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +""" + tasks.filemap + ~~~~~~~~~~~~~ + + tests/filename_map.yml validity checks +""" +import pathlib +import re + +import yaml +from invoke import task # pylint: disable=3rd-party-module-not-gated +from tasks import utils + +CODE_DIR = pathlib.Path(__file__).resolve().parent.parent +FILENAME_MAP_PATH = CODE_DIR / "tests" / "filename_map.yml" + + +def _match_to_test_file(match): + tests_path = CODE_DIR / "tests" + parts = match.split(".") + parts[-1] += ".py" + return tests_path.joinpath(*parts).relative_to(CODE_DIR) + + +def _check_matches(rule, matches): + errors = 0 + for match in matches: + filematch = _match_to_test_file(match) + if not filematch.exists(): + utils.error( + "The match '{}' for rule '{}' points to a non existing test module path: {}", + match, + rule, + filematch, + ) + errors += 1 + return errors + + +@task +def check(ctx): + exitcode = 0 + excludes = ("tasks/", "templates/", ".nox/") + full_filelist = [path.relative_to(CODE_DIR) for path in CODE_DIR.rglob("*.py")] + filelist = [ + str(path) for path in full_filelist if not str(path).startswith(excludes) + ] + filename_map = yaml.safe_load(FILENAME_MAP_PATH.read_text()) + checked = set() + for rule, matches in filename_map.items(): + if rule == "*": + exitcode += _check_matches(rule, matches) + elif "|" in rule: + # This is regex + for filepath in filelist: + if re.match(rule, filepath): + # Found at least one match, stop looking + break + else: + utils.error( + "Could not find a matching file in the salt repo for the rule '{}'", + rule, + ) + exitcode += 1 + continue + exitcode += _check_matches(rule, matches) + elif "*" in rule or "\\" in rule: + # Glob matching + process_matches = True + for filerule in CODE_DIR.glob(rule): + if not filerule.exists(): + utils.error( + "The rule '{}' points to a non existing path: {}", + rule, + filerule, + ) + exitcode += 1 + process_matches = False + if process_matches: + exitcode += _check_matches(rule, matches) + else: + # Direct file paths as rules + filerule = pathlib.Path(rule) + if not filerule.exists(): + utils.error( + "The rule '{}' points to a non existing path: {}", rule, filerule + ) + exitcode += 1 + continue + exitcode += _check_matches(rule, matches) + if exitcode: + utils.error("Found {} errors", exitcode) + utils.exit_invoke(exitcode)