Allow knowing salt's internal loaders paths

salt/loader/__init__.py

@@ -64,6 +64,41 @@ LIBCLOUD_FUNCS_NOT_SUPPORTED = (
     "proxmox.avail_sizes",
 )
 
+SALT_INTERNAL_LOADERS_PATHS = (
+    str(SALT_BASE_PATH / "auth"),
+    str(SALT_BASE_PATH / "beacons"),
+    str(SALT_BASE_PATH / "cache"),
+    str(SALT_BASE_PATH / "client" / "ssh" / "wrapper"),
+    str(SALT_BASE_PATH / "cloud" / "clouds"),
+    str(SALT_BASE_PATH / "engines"),
+    str(SALT_BASE_PATH / "executors"),
+    str(SALT_BASE_PATH / "fileserver"),
+    str(SALT_BASE_PATH / "grains"),
+    str(SALT_BASE_PATH / "log" / "handlers"),
+    str(SALT_BASE_PATH / "matchers"),
+    str(SALT_BASE_PATH / "metaproxy"),
+    str(SALT_BASE_PATH / "modules"),
+    str(SALT_BASE_PATH / "netapi"),
+    str(SALT_BASE_PATH / "output"),
+    str(SALT_BASE_PATH / "pillar"),
+    str(SALT_BASE_PATH / "proxy"),
+    str(SALT_BASE_PATH / "queues"),
+    str(SALT_BASE_PATH / "renderers"),
+    str(SALT_BASE_PATH / "returners"),
+    str(SALT_BASE_PATH / "roster"),
+    str(SALT_BASE_PATH / "runners"),
+    str(SALT_BASE_PATH / "sdb"),
+    str(SALT_BASE_PATH / "serializers"),
+    str(SALT_BASE_PATH / "spm" / "pkgdb"),
+    str(SALT_BASE_PATH / "spm" / "pkgfiles"),
+    str(SALT_BASE_PATH / "states"),
+    str(SALT_BASE_PATH / "thorium"),
+    str(SALT_BASE_PATH / "tokens"),
+    str(SALT_BASE_PATH / "tops"),
+    str(SALT_BASE_PATH / "utils"),
+    str(SALT_BASE_PATH / "wheel"),
+)
+
 
 def static_loader(
     opts,
@@ -110,9 +145,16 @@ def _module_dirs(
 ):
     if tag is None:
         tag = ext_type
-    sys_types = os.path.join(base_path or SALT_BASE_PATH, int_type or ext_type)
+    sys_types = os.path.join(base_path or str(SALT_BASE_PATH), int_type or ext_type)
     ext_types = os.path.join(opts["extension_modules"], ext_type)
 
+    if not sys_types.startswith(SALT_INTERNAL_LOADERS_PATHS):
+        raise RuntimeError(
+            "{!r} is not considered a salt internal loader path. If this "
+            "is a new loader being added, please also add it to "
+            "{}.SALT_INTERNAL_LOADERS_PATHS.".format(sys_types, __name__)
+        )
+
     ext_type_types = []
     if ext_dirs:
         if ext_type_dirs is None:
@@ -662,7 +704,7 @@ def log_handlers(opts):
             opts,
             "log_handlers",
             int_type="handlers",
-            base_path=os.path.join(SALT_BASE_PATH, "log"),
+            base_path=str(SALT_BASE_PATH / "log"),
         ),
         opts,
         tag="log_handlers",
@@ -678,7 +720,7 @@ def ssh_wrapper(opts, functions=None, context=None):
         _module_dirs(
             opts,
             "wrapper",
-            base_path=os.path.join(SALT_BASE_PATH, os.path.join("client", "ssh")),
+            base_path=str(SALT_BASE_PATH / "client" / "ssh"),
         ),
         opts,
         tag="wrapper",
@@ -1010,7 +1052,7 @@ def call(fun, **kwargs):
     dirs = kwargs.get("dirs", [])
 
     funcs = LazyLoader(
-        [os.path.join(SALT_BASE_PATH, "modules")] + dirs,
+        [str(SALT_BASE_PATH / "modules")] + dirs,
         None,
         tag="modules",
         virtual_enable=False,
@@ -1077,7 +1119,7 @@ def pkgdb(opts):
     .. versionadded:: 2015.8.0
     """
     return LazyLoader(
-        _module_dirs(opts, "pkgdb", base_path=os.path.join(SALT_BASE_PATH, "spm")),
+        _module_dirs(opts, "pkgdb", base_path=str(SALT_BASE_PATH / "spm")),
         opts,
         tag="pkgdb",
     )
@@ -1090,7 +1132,7 @@ def pkgfiles(opts):
     .. versionadded:: 2015.8.0
     """
     return LazyLoader(
-        _module_dirs(opts, "pkgfiles", base_path=os.path.join(SALT_BASE_PATH, "spm")),
+        _module_dirs(opts, "pkgfiles", base_path=str(SALT_BASE_PATH / "spm")),
         opts,
         tag="pkgfiles",
     )
@@ -1100,7 +1142,7 @@ def clouds(opts):
     """
     Return the cloud functions
     """
-    _utils = salt.loader.utils(opts)
+    _utils = utils(opts)
     # Let's bring __active_provider_name__, defaulting to None, to all cloud
     # drivers. This will get temporarily updated/overridden with a context
     # manager when needed.
@@ -1109,7 +1151,7 @@ def clouds(opts):
             opts,
             "clouds",
             "cloud",
-            base_path=os.path.join(SALT_BASE_PATH, "cloud"),
+            base_path=str(SALT_BASE_PATH / "cloud"),
             int_type="clouds",
         ),
         opts,

salt/loader/lazy.py

@@ -1,12 +1,12 @@
-import contextvars
 import copy
 import functools
 import importlib
-import importlib.machinery  # pylint: disable=no-name-in-module,import-error
-import importlib.util  # pylint: disable=no-name-in-module,import-error
+import importlib.machinery
+import importlib.util
 import inspect
 import logging
 import os
+import pathlib
 import re
 import sys
 import tempfile
@@ -35,6 +35,15 @@ import salt.utils.stringutils
 import salt.utils.versions
 from salt.utils.decorators import Depends
 
+try:
+    # Try the stdlib C extension first
+    import _contextvars as contextvars
+except ImportError:
+    # Py<3.7
+    import contextvars
+
+log = logging.getLogger(__name__)
+
 # pylint: disable=no-member
 MODULE_KIND_SOURCE = 1
 MODULE_KIND_COMPILED = 2
@@ -55,15 +64,13 @@ MODULE_KIND_MAP = {
 # pylint: enable=no-member
 
 
-SALT_BASE_PATH = os.path.abspath(salt.syspaths.INSTALL_DIR)
+SALT_BASE_PATH = pathlib.Path(salt.syspaths.INSTALL_DIR).resolve()
 LOADED_BASE_NAME = "salt.loaded"
 PY3_PRE_EXT = re.compile(r"\.cpython-{}{}(\.opt-[1-9])?".format(*sys.version_info[:2]))
 
 # Will be set to pyximport module at runtime if cython is enabled in config.
 pyximport = None
 
-log = logging.getLogger(__name__)
-
 
 def _generate_module(name):
     if name in sys.modules:
@@ -77,7 +84,7 @@ def _generate_module(name):
 
 
 def _mod_type(module_path):
-    if module_path.startswith(SALT_BASE_PATH):
+    if module_path.startswith(str(SALT_BASE_PATH)):
         return "int"
     return "ext"
 

tests/pytests/unit/loader/test_lazy.py

@@ -1,9 +1,6 @@
 """
 Tests for salt.loader.lazy
 """
-
-import os
-import shutil
 import sys
 
 import pytest
@@ -11,7 +8,6 @@ import salt.loader
 import salt.loader.context
 import salt.loader.lazy
 import salt.utils.files
-from tests.support.helpers import dedent
 
 
 @pytest.fixture
@@ -20,8 +16,7 @@ def loader_dir(tmp_path):
     Create a simple directory with a couple modules to load and run tests
     against.
     """
-    mod_content = dedent(
-        """
+    mod_contents = """
     def __virtual__():
         return True
 
@@ -31,16 +26,10 @@ def loader_dir(tmp_path):
     def get_context(key):
         return __context__[key]
     """
-    )
-    tmp_path = str(tmp_path)
-    with salt.utils.files.fopen(os.path.join(tmp_path, "mod_a.py"), "w") as fp:
-        fp.write(mod_content)
-    with salt.utils.files.fopen(os.path.join(tmp_path, "mod_b.py"), "w") as fp:
-        fp.write(mod_content)
-    try:
-        yield tmp_path
-    finally:
-        shutil.rmtree(tmp_path)
+    with pytest.helpers.temp_file(
+        "mod_a.py", directory=tmp_path, contents=mod_contents
+    ), pytest.helpers.temp_file("mod_b.py", directory=tmp_path, contents=mod_contents):
+        yield str(tmp_path)
 
 
 def test_loaders_have_uniq_context(loader_dir):
@@ -116,3 +105,10 @@ def test_loaders_convert_context_to_values(loader_dir):
     assert loader_1.opts["grains"] == grains_default
     # The loader's opts is a copy
     assert opts["grains"] == grains
+
+
+def test_missing_loader_from_salt_internal_loaders():
+    with pytest.raises(RuntimeError):
+        salt.loader._module_dirs(
+            {"extension_modules": "/tmp/foo"}, "missingmodules", "module"
+        )