add __env__ substitution inside file and pillar root paths

changelog/55747.added

@@ -0,0 +1 @@
+Add __env__ substitution inside file and pillar root paths

doc/ref/configuration/master.rst

@@ -2871,6 +2871,8 @@ roots: Master's Local File Server
 ``file_roots``
 **************
 
+.. versionchanged:: 3005
+
 Default:
 
 .. code-block:: yaml
@@ -2905,6 +2907,29 @@ Example:
       __env__:
         - /srv/salt/default
 
+Taking dynamic environments one step further, ``__env__`` can also be used in
+the ``file_roots`` filesystem path as of version 3005. It will be replaced with
+the actual ``saltenv`` and searched for states and data to provide to the
+minion. For instance, this configuration:
+
+.. code-block:: yaml
+
+    file_roots:
+      __env__:
+        - /srv/__env__/salt
+
+is equivalent to this static configuration:
+
+.. code-block:: yaml
+
+    file_roots:
+      dev:
+        - /srv/dev/salt
+      test:
+        - /srv/test/salt
+      prod:
+        - /srv/prod/salt
+
 .. note::
     For masterless Salt, this parameter must be specified in the minion config
     file.
@@ -4010,6 +4035,8 @@ Pillar Configuration
 ``pillar_roots``
 ----------------
 
+.. versionchanged:: 3005
+
 Default:
 
 .. code-block:: yaml
@@ -4036,6 +4063,29 @@ Example:
       __env__:
         - /srv/pillar/others
 
+Taking dynamic environments one step further, ``__env__`` can also be used in
+the ``pillar_roots`` filesystem path as of version 3005. It will be replaced
+with the actual ``pillarenv`` and searched for Pillar data to provide to the
+minion. For instance, this configuration:
+
+.. code-block:: yaml
+
+    pillar_roots:
+      __env__:
+        - /srv/__env__/pillar
+
+is equivalent to this static configuration:
+
+.. code-block:: yaml
+
+    pillar_roots:
+      dev:
+        - /srv/dev/pillar
+      test:
+        - /srv/test/pillar
+      prod:
+        - /srv/prod/pillar
+
 .. conf_master:: on_demand_ext_pillar
 
 ``on_demand_ext_pillar``

doc/topics/pillar/index.rst

@@ -216,6 +216,30 @@ pillar applying to all environments. For example:
 
 .. versionadded:: 2017.7.5,2018.3.1
 
+Taking it one step further, ``__env__`` can also be used in the ``pillar_root``
+filesystem path. It will be replaced with the actual ``pillarenv`` and searched
+for Pillar data to provide to the minion. For instance, this configuration:
+
+.. code-block:: yaml
+
+    pillar_roots:
+      __env__:
+        - /srv/__env__/pillar
+
+is equivalent to this static configuration:
+
+.. code-block:: yaml
+
+    pillar_roots:
+      dev:
+        - /srv/dev/pillar
+      test:
+        - /srv/test/pillar
+      prod:
+        - /srv/prod/pillar
+
+.. versionadded:: 3005
+
 
 Pillar Namespace Flattening
 ===========================

salt/fileserver/roots.py

@@ -36,6 +36,7 @@ def find_file(path, saltenv="base", **kwargs):
     """
     Search the environment for the relative path.
     """
+    actual_saltenv = saltenv
     if "env" in kwargs:
         # "env" is not supported; Use "saltenv".
         kwargs.pop("env")
@@ -94,6 +95,7 @@ def find_file(path, saltenv="base", **kwargs):
             return _add_file_stat(fnd)
         return fnd
     for root in __opts__["file_roots"][saltenv]:
+        root = root.replace("__env__", actual_saltenv)
         full = os.path.join(root, path)
         if os.path.isfile(full) and not salt.fileserver.is_file_ignored(__opts__, full):
             fnd["path"] = full
@@ -305,6 +307,7 @@ def _file_lists(load, form):
         load.pop("env")
 
     saltenv = load["saltenv"]
+    actual_saltenv = saltenv
     if saltenv not in __opts__["file_roots"]:
         if "__env__" in __opts__["file_roots"]:
             log.debug(
@@ -404,6 +407,7 @@ def _file_lists(load, form):
                             ret["links"][rel_path] = link_dest
 
         for path in __opts__["file_roots"][saltenv]:
+            path = path.replace("__env__", actual_saltenv)
             for root, dirs, files in salt.utils.path.os_walk(
                 path, followlinks=__opts__["fileserver_followsymlinks"]
             ):

salt/pillar/__init__.py

@@ -657,6 +657,11 @@ class Pillar:
                     env,
                 )
                 opts["pillar_roots"].pop("__env__")
+        for env in opts["pillar_roots"]:
+            for idx, root in enumerate(opts["pillar_roots"][env]):
+                opts["pillar_roots"][env][idx] = opts["pillar_roots"][env][idx].replace(
+                    "__env__", env
+                )
         return opts
 
     def _get_envs(self):

tests/pytests/functional/fileserver/test_roots.py

@@ -1,6 +1,8 @@
 import pytest
 import salt.config
 import salt.fileserver.roots as roots
+from salt.utils.odict import OrderedDict
+from tests.support.mock import patch
 
 pytestmark = [
     pytest.mark.windows_whitelisted,
@@ -24,3 +26,66 @@ def test_symlink_list(base_env_state_tree_root_dir):
         link.symlink_to(str(target))
         ret = roots.symlink_list({"saltenv": "base"})
         assert ret == {"link": str(target)}
+
+
+@pytest.mark.parametrize(
+    "env",
+    ("base", "something-else", "cool_path_123"),
+)
+def test_fileserver_roots_find_file_envs_path_substitution(
+    env, temp_salt_minion, tmp_path
+):
+    """
+    Test fileserver access to a dynamic path using __env__
+    """
+    fn = "test.txt"
+    opts = temp_salt_minion.config.copy()
+
+    envpath = tmp_path / env
+    envpath.mkdir(parents=True, exist_ok=True)
+    filepath = envpath / fn
+    filepath.touch()
+
+    # Stop using OrderedDict once we drop Py3.5 support
+    expected = OrderedDict()
+    expected["rel"] = fn
+    expected["path"] = str(filepath)
+
+    # Stop using OrderedDict once we drop Py3.5 support
+    opts["file_roots"] = OrderedDict()
+    opts["file_roots"][env] = [str(tmp_path / "__env__")]
+
+    with patch("salt.fileserver.roots.__opts__", opts, create=True):
+        ret = roots.find_file(fn, saltenv=env)
+    ret.pop("stat")
+    assert ret == expected
+
+
+@pytest.mark.parametrize(
+    "env",
+    ("base", "something-else", "cool_path_123"),
+)
+def test_fileserver_roots__file_lists_envs_path_substitution(
+    env, temp_salt_minion, tmp_path
+):
+    """
+    Test fileserver access to a dynamic path using __env__
+    """
+    fn = "test.txt"
+    opts = temp_salt_minion.config.copy()
+
+    envpath = tmp_path / env
+    envpath.mkdir(parents=True, exist_ok=True)
+    filepath = envpath / fn
+    filepath.touch()
+
+    expected = [fn]
+
+    # Stop using OrderedDict once we drop Py3.5 support
+    opts["file_roots"] = OrderedDict()
+    opts["file_roots"][env] = [str(tmp_path / "__env__")]
+
+    with patch("salt.fileserver.roots.__opts__", opts, create=True):
+        ret = roots._file_lists({"saltenv": env}, "files")
+
+    assert ret == expected

tests/pytests/unit/pillar/test_pillar.py

@@ -43,3 +43,27 @@ def test_pillar_get_tops_should_not_error_when_merging_strategy_is_none_and_no_p
     )
     tops, errors = pillar.get_tops()
     assert not errors
+
+
+@pytest.mark.parametrize(
+    "env",
+    ("base", "something-else", "cool_path_123"),
+)
+def test_pillar_envs_path_substitution(env, temp_salt_minion, tmp_path):
+    """
+    Test pillar access to a dynamic path using __env__
+    """
+    opts = temp_salt_minion.config.copy()
+    expected = {env: [str(tmp_path / env)]}
+    # Stop using OrderedDict once we drop Py3.5 support
+    opts["pillar_roots"] = OrderedDict()
+    opts["pillar_roots"][env] = [str(tmp_path / "__env__")]
+    grains = salt.loader.grains(opts)
+    pillar = salt.pillar.Pillar(
+        opts,
+        grains,
+        temp_salt_minion.id,
+        env,
+    )
+    # The __env__ string in the path has been substituted for the actual env
+    assert pillar.opts["pillar_roots"] == expected