Migrate tasks/loader.py -> tools/precommit/loader.py

Refs #64374

Signed-off-by: Pedro Algarvio <palgarvio@vmware.com>

.pre-commit-config.yaml

@@ -115,6 +115,20 @@ repos:
           - docstrings
           - check
 
+      - id: tools
+        alias: loader-check-virtual
+        name: Check loader modules __virtual__
+        files: salt/.*\.py$
+        exclude: >
+            (?x)^(
+                templates/.*|
+                salt/ext/.*|
+            )$
+        args:
+          - pre-commit
+          - salt-loaders
+          - check-virtual
+
   # ----- Packaging Requirements ------------------------------------------------------------------------------------>
 
   - repo: https://github.com/saltstack/pip-tools-compile-impersonate
@@ -1247,26 +1261,6 @@ repos:
           - packaging
           - looseversion
 
-      - id: invoke
-        alias: loader-check-virtual
-        name: Check loader modules __virtual__
-        files: salt/.*\.py$
-        exclude: >
-            (?x)^(
-                templates/.*|
-                salt/ext/.*|
-            )$
-        args:
-          - loader.check-virtual
-        additional_dependencies:
-          - blessings==1.7
-          - pyyaml==6.0.1
-          - distro==1.7.0
-          - jinja2==3.0.3
-          - msgpack==1.0.3
-          - packaging
-          - looseversion
-
   - repo: https://github.com/saltstack/invoke-pre-commit
     rev: v1.9.0
     hooks:

changelog/64374.fixed.md

@@ -2,3 +2,4 @@ Migrated some [`invoke`](https://www.pyinvoke.org/) tasks to [`python-tools-scri
 
 * `tasks/docs.py` -> `tools/precommit/docs.py`
 * `tasks/docstrings.py` -> `tools/precommit/docstrings.py`
+* `tasks/loader.py` -> `tools/precommit/loader.py`

tools/__init__.py

@@ -44,6 +44,7 @@ ptscripts.register_tools_module("tools.precommit.changelog")
 ptscripts.register_tools_module("tools.precommit.workflows")
 ptscripts.register_tools_module("tools.precommit.docs")
 ptscripts.register_tools_module("tools.precommit.docstrings")
+ptscripts.register_tools_module("tools.precommit.loader")
 ptscripts.register_tools_module("tools.release", venv_config=RELEASE_VENV_CONFIG)
 ptscripts.register_tools_module("tools.testsuite")
 ptscripts.register_tools_module("tools.testsuite.download")

tools/precommit/__init__.py

@@ -3,7 +3,47 @@ These commands, and sub-commands, are used by pre-commit.
 """
 from ptscripts import command_group
 
+import tools.utils
+
 # Define the command group
 cgroup = command_group(
     name="pre-commit", help="Pre-Commit Related Commands", description=__doc__
 )
+
+SALT_BASE_PATH = tools.utils.REPO_ROOT / "salt"
+
+SALT_INTERNAL_LOADERS_PATHS = (
+    # This is a 1:1 copy of SALT_INTERNAL_LOADERS_PATHS found in salt/loader/__init__.py
+    str(SALT_BASE_PATH / "auth"),
+    str(SALT_BASE_PATH / "beacons"),
+    str(SALT_BASE_PATH / "cache"),
+    str(SALT_BASE_PATH / "client" / "ssh" / "wrapper"),
+    str(SALT_BASE_PATH / "cloud" / "clouds"),
+    str(SALT_BASE_PATH / "engines"),
+    str(SALT_BASE_PATH / "executors"),
+    str(SALT_BASE_PATH / "fileserver"),
+    str(SALT_BASE_PATH / "grains"),
+    str(SALT_BASE_PATH / "log_handlers"),
+    str(SALT_BASE_PATH / "matchers"),
+    str(SALT_BASE_PATH / "metaproxy"),
+    str(SALT_BASE_PATH / "modules"),
+    str(SALT_BASE_PATH / "netapi"),
+    str(SALT_BASE_PATH / "output"),
+    str(SALT_BASE_PATH / "pillar"),
+    str(SALT_BASE_PATH / "proxy"),
+    str(SALT_BASE_PATH / "queues"),
+    str(SALT_BASE_PATH / "renderers"),
+    str(SALT_BASE_PATH / "returners"),
+    str(SALT_BASE_PATH / "roster"),
+    str(SALT_BASE_PATH / "runners"),
+    str(SALT_BASE_PATH / "sdb"),
+    str(SALT_BASE_PATH / "serializers"),
+    str(SALT_BASE_PATH / "spm" / "pkgdb"),
+    str(SALT_BASE_PATH / "spm" / "pkgfiles"),
+    str(SALT_BASE_PATH / "states"),
+    str(SALT_BASE_PATH / "thorium"),
+    str(SALT_BASE_PATH / "tokens"),
+    str(SALT_BASE_PATH / "tops"),
+    str(SALT_BASE_PATH / "utils"),
+    str(SALT_BASE_PATH / "wheel"),
+)

tools/precommit/docstrings.py

@@ -16,8 +16,8 @@ from typing import TYPE_CHECKING
 from ptscripts import Context, command_group
 
 import tools.utils
-from salt.loader import SALT_INTERNAL_LOADERS_PATHS
 from salt.version import SaltStackVersion
+from tools.precommit import SALT_INTERNAL_LOADERS_PATHS
 
 SALT_CODE_DIR = tools.utils.REPO_ROOT / "salt"
 SALT_MODULES_PATH = SALT_CODE_DIR / "modules"
@@ -865,7 +865,7 @@ def check_docstrings(
     Check salt's docstrings
     """
     if not files:
-        _files = SALT_CODE_DIR.rglob("*.py")
+        _files = list(SALT_CODE_DIR.rglob("*.py"))
     else:
         _files = [fpath.resolve() for fpath in files if fpath.suffix == ".py"]
 

tools/precommit/loader.py

@@ -1,24 +1,35 @@
 """
-    tasks.loader
+Salt loader checks
-    ~~~~~~~~~~~~
-
-    Salt loader checks
 """
 
 import ast
 import pathlib
 
-from invoke import task  # pylint: disable=3rd-party-module-not-gated
+from ptscripts import Context, command_group
 
-from salt.loader import SALT_INTERNAL_LOADERS_PATHS
+import tools.utils
-from tasks import utils
+from tools.precommit import SALT_INTERNAL_LOADERS_PATHS
 
-CODE_DIR = pathlib.Path(__file__).resolve().parent.parent
+SALT_CODE_DIR = tools.utils.REPO_ROOT / "salt"
-SALT_CODE_DIR = CODE_DIR / "salt"
+
+cgroup = command_group(name="salt-loaders", help=__doc__, parent="pre-commit")
 
 
-@task(iterable=["files"], positional=["files"])
+@cgroup.command(
-def check_virtual(ctx, files, enforce_virtualname=False):
+    name="check-virtual",
+    arguments={
+        "files": {
+            "help": "List of files to check",
+            "nargs": "*",
+        },
+        "enforce_virtualname": {
+            "help": "Enforce the usage of `__virtualname__`",
+        },
+    },
+)
+def check_virtual(
+    ctx: Context, files: list[pathlib.Path], enforce_virtualname: bool = False
+) -> None:
     """
     Check Salt loader modules for a defined `__virtualname__` attribute and `__virtual__` function.
 
@@ -26,23 +37,10 @@ def check_virtual(ctx, files, enforce_virtualname=False):
 
         https://github.com/saltstack/salt/blob/27ae8260983b11fe6e32a18e777d550be9fe1dc2/tests/unit/test_virtualname.py
     """
-    # CD into Salt's repo root directory
+    if not files:
-    ctx.cd(CODE_DIR)
+        _files = list(SALT_CODE_DIR.rglob("*.py"))
-
-    # Unfortunately invoke does not support nargs.
-    # We migth have been passed --files="foo.py bar.py"
-    # Turn that into a list of paths
-    _files = []
-    for path in files:
-        if not path:
-            continue
-        _files.extend(path.split())
-    if not _files:
-        _files = SALT_CODE_DIR.rglob("*.py")
     else:
-        _files = [pathlib.Path(fname) for fname in _files]
+        _files = [fpath.resolve() for fpath in files if fpath.suffix == ".py"]
-
-    _files = [path.resolve() for path in _files]
 
     errors = 0
     exitcode = 0
@@ -78,14 +76,15 @@ def check_virtual(ctx, files, enforce_virtualname=False):
                             continue
                         if target.id == "__virtualname__":
                             found_virtualname_attr = True
-                            if node.value.s not in path.name:
+                            if node.value.s not in path.name:  # type: ignore[attr-defined]
                                 errors += 1
                                 exitcode = 1
-                                utils.error(
+                                ctx.error(
                                     'The value of the __virtualname__ attribute, "{}"'
-                                    " is not part of {}",
+                                    " is not part of {}".format(
-                                    node.value.s,
+                                        node.value.s,  # type: ignore[attr-defined]
-                                    path.name,
+                                        path.name,
+                                    )
                                 )
             if found_virtualname_attr:
                 break
@@ -93,11 +92,10 @@ def check_virtual(ctx, files, enforce_virtualname=False):
         if not found_virtualname_attr and enforce_virtualname:
             errors += 1
             exitcode = 1
-            utils.error(
+            ctx.error(
-                "The salt loader module {} defines a __virtual__() function but does"
+                f"The salt loader module {path.relative_to(tools.utils.REPO_ROOT)} defines "
-                " not define a __virtualname__ attribute",
+                "a __virtual__() function but does not define a __virtualname__ attribute"
-                path.relative_to(CODE_DIR),
             )
     if exitcode:
-        utils.error("Found {} errors", errors)
+        ctx.error(f"Found {errors} errors")
-    utils.exit_invoke(exitcode)
+    ctx.exit(exitcode)