Merge 3007.x into master

changelog/63060.fixed.md

@@ -0,0 +1 @@
+file.managed correctly handles file path with '#'

changelog/66179.fixed.md

@@ -0,0 +1 @@
+Log "Publish server binding pub to" messages to debug instead of error level.

changelog/66264.fixed.md

@@ -0,0 +1 @@
+Fix cache directory setting in Master Cluster tutorial

changelog/66266.fixed.md

@@ -0,0 +1 @@
+Change log level of successful master cluster key exchange from error to info.

changelog/66290.fixed.md

@@ -0,0 +1,2 @@
+Chocolatey: Make sure the return dictionary from ``chocolatey.version``
+contains lowercase keys

changelog/66292.fixed.md

@@ -0,0 +1 @@
+fix cacheing inline pillar, by not rendering inline pillar during cache save function.

doc/topics/tutorials/master-cluster.rst

@@ -91,7 +91,7 @@ Master Config:
           - 10.27.7.126
           - 10.27.3.73
         cluster_pki_dir: /my/gluster/share/pki
-        cache_dir: /my/gluster/share/cache
+        cachedir: /my/gluster/share/cache
         file_roots:
             - /my/gluster/share/srv/salt
         pillar_roots:

salt/channel/server.py

@@ -1074,7 +1074,7 @@ class MasterPubServerChannel:
                 if m_digest != digest:
                     log.error("Invalid aes signature from peer: %s", peer)
                     return
-                log.error("Received new key from peer %s", peer)
+                log.info("Received new key from peer %s", peer)
                 if peer in self.peer_keys:
                     if self.peer_keys[peer] != key_str:
                         self.peer_keys[peer] = key_str

salt/fileclient.py

@@ -478,7 +478,7 @@ class Client:
         """
         Get a single file from a URL.
         """
-        url_data = urllib.parse.urlparse(url)
+        url_data = urllib.parse.urlparse(url, allow_fragments=False)
         url_scheme = url_data.scheme
         url_path = os.path.join(url_data.netloc, url_data.path).rstrip(os.sep)
 

salt/modules/chocolatey.py

@@ -1204,8 +1204,8 @@ def version(name, check_remote=False, source=None, pre_versions=False):
     if installed:
         for pkg in installed:
             if lower_name == pkg.lower():
-                packages.setdefault(pkg, {})
-                packages[pkg]["installed"] = installed[pkg]
+                packages.setdefault(lower_name, {})
+                packages[lower_name]["installed"] = installed[pkg]
 
     if check_remote:
         # If there's a remote package available, then also include that
@@ -1216,8 +1216,8 @@ def version(name, check_remote=False, source=None, pre_versions=False):
         if available:
             for pkg in available:
                 if lower_name == pkg.lower():
-                    packages.setdefault(pkg, {})
-                    packages[pkg]["available"] = available[pkg]
+                    packages.setdefault(lower_name, {})
+                    packages[lower_name]["available"] = available[pkg]
 
     return packages
 

salt/pillar/__init__.py

@@ -474,7 +474,7 @@ class PillarCache:
             self.saltenv,
             ext=self.ext,
             functions=self.functions,
-            pillar_override=self.pillar_override,
+            pillar_override=None,
             pillarenv=self.pillarenv,
             extra_minion_data=self.extra_minion_data,
         )

salt/runners/winrepo.py

@@ -174,7 +174,11 @@ def update_git_repos(opts=None, clean=False, masterless=False):
                     rev, remote_url = remote_info.strip().split()
                 except ValueError:
                     remote_url = remote_info
-                gittarget = os.path.join(base_dir, targetname).replace(".", "_")
+                targetname = targetname.replace(".git", "")
+                # GitFS using pygit2 and gitpython place the repo in a
+                # subdirectory named `_`. We need to stay consistent when using
+                # the legacy method as well
+                gittarget = os.path.join(base_dir, targetname, "_")
                 if masterless:
                     result = __salt__["state.single"](
                         "git.latest",

salt/transport/tcp.py

@@ -1403,12 +1403,12 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer):
             ssl=ctx,
         )
         if self.pub_path:
-            log.error(
+            log.debug(
                 "Publish server binding pub to %s ssl=%r", self.pub_path, self.ssl
             )
             sock = tornado.netutil.bind_unix_socket(self.pub_path)
         else:
-            log.error(
+            log.debug(
                 "Publish server binding pub to %s:%s ssl=%r",
                 self.pub_host,
                 self.pub_port,

salt/transport/ws.py

@@ -131,7 +131,7 @@ class PublishClient(salt.transport.base.PublishClient):
                         url = "https://ipc.saltproject.io/ws"
                     else:
                         url = "http://ipc.saltproject.io/ws"
-                log.error("pub client connect %r %r", url, ctx)
+                log.debug("pub client connect %r %r", url, ctx)
                 ws = await asyncio.wait_for(session.ws_connect(url, ssl=ctx), 3)
             except Exception as exc:  # pylint: disable=broad-except
                 log.warning(
@@ -397,7 +397,7 @@ class PublishServer(salt.transport.base.DaemonizedPublishServer):
         else:
             if cert:
                 name = salt.transport.base.common_name(cert)
-                log.error("Request client cert %r", name)
+                log.debug("Request client cert %r", name)
         ws = aiohttp.web.WebSocketResponse()
         await ws.prepare(request)
         self.clients.add(ws)
@@ -512,7 +512,7 @@ class RequestServer(salt.transport.base.DaemonizedRequestServer):
         else:
             if cert:
                 name = salt.transport.base.common_name(cert)
-                log.error("Request client cert %r", name)
+                log.debug("Request client cert %r", name)
         ws = aiohttp.web.WebSocketResponse()
         await ws.prepare(request)
         async for msg in ws:
@@ -550,7 +550,7 @@ class RequestClient(salt.transport.base.RequestClient):
             ctx = tornado.netutil.ssl_options_to_context(self.ssl, server_side=False)
         self.session = aiohttp.ClientSession()
         URL = self.get_master_uri(self.opts)
-        log.error("Connect to %s %s", URL, ctx)
+        log.debug("Connect to %s %s", URL, ctx)
         self.ws = await self.session.ws_connect(URL, ssl=ctx)
 
     async def send(self, load, timeout=60):

tests/pytests/functional/runners/test_winrepo.py

@@ -1,3 +1,5 @@
+import os
+
 import pytest
 
 from salt.runners import winrepo
@@ -10,27 +12,21 @@ pytestmark = [
 
 @pytest.fixture
 def configure_loader_modules(minion_opts, tmp_path):
-    opts = minion_opts.copy()
     winrepo_dir = tmp_path / "winrepo"
     winrepo_dir.mkdir()
     winrepo_dir_ng = tmp_path / "winrepo_ng"
     winrepo_dir_ng.mkdir()
-    opts["winrepo_dir"] = str(winrepo_dir)
-    opts["winrepo_dir_ng"] = str(winrepo_dir_ng)
-
-    return {
-        winrepo: {
-            "__opts__": opts,
-        }
-    }
+    minion_opts["winrepo_dir"] = str(winrepo_dir)
+    minion_opts["winrepo_dir_ng"] = str(winrepo_dir_ng)
+    return {winrepo: {"__opts__": minion_opts}}
 
 
 @pytest.fixture
 def winrepo_remotes(minion_opts):
-    winrepo_remotes = minion_opts.get("winrepo_remotes", [])
-    winrepo_remotes_ng = minion_opts.get("winrepo_remotes_ng", [])
-    winrepo_remotes.extend(winrepo_remotes_ng)
-    return winrepo_remotes
+    remotes = set()
+    remotes.update(minion_opts.get("winrepo_remotes", []))
+    remotes.update(minion_opts.get("winrepo_remotes_ng", []))
+    return remotes
 
 
 def test_update_git_repos(winrepo_remotes):
@@ -38,15 +34,19 @@ def test_update_git_repos(winrepo_remotes):
     Ensure update git repos works as intended.
     """
     res = winrepo.update_git_repos()
-
     assert res
 
     for remote in winrepo_remotes:
         assert remote in res
         assert res[remote]
 
+        # Make sure there are package definitions in the root
+        assert res[remote].endswith("_")
+        pkg_def = os.path.join(res[remote], "7zip.sls")
+        assert os.path.exists(pkg_def)
 
-def test_legacy_update_git_repos(winrepo_remotes):
+
+def test_legacy_update_git_repos(winrepo_remotes, minion_opts):
     """
     Ensure update git repos works as intended with legacy (non-gitfs) code.
     """
@@ -58,3 +58,15 @@ def test_legacy_update_git_repos(winrepo_remotes):
         for remote in winrepo_remotes:
             assert remote in res
             assert res[remote]
+
+            # Make sure there are package definitions in the root
+            # We have to look up the actual repo dir here because the legacy
+            # update only returns True or False, not a path
+            if "-ng" in remote:
+                path = minion_opts["winrepo_dir_ng"]
+                pkg_def = os.path.join(path, "salt-winrepo-ng", "_", "7zip.sls")
+            else:
+                path = minion_opts["winrepo_dir"]
+                pkg_def = os.path.join(path, "salt-winrepo", "_", "7zip.sls")
+
+            assert os.path.exists(pkg_def)

tests/pytests/functional/states/chocolatey/test_post_20.py

@@ -109,6 +109,13 @@ def vim(chocolatey_mod):
     chocolatey_mod.uninstall(name="vim", force=True)
 
 
+@pytest.fixture(scope="function")
+def everything(chocolatey_mod):
+    chocolatey_mod.install(name="everything", version="1.4.1935")
+    yield
+    chocolatey_mod.uninstall(name="everything", force=True)
+
+
 def test_installed_latest(clean, chocolatey, chocolatey_mod):
     chocolatey.installed(name="vim")
     result = chocolatey_mod.version(name="vim")
@@ -122,6 +129,14 @@ def test_installed_version(clean, chocolatey, chocolatey_mod):
     assert result["vim"]["installed"][0] == "9.0.1672"
 
 
+def test_installed_version_existing_capitalization(
+    everything, chocolatey, chocolatey_mod
+):
+    result = chocolatey.installed(name="everything", version="1.4.11024")
+    expected_changes = {"Everything": {"new": ["1.4.11024"], "old": ["1.4.1935"]}}
+    assert result["changes"] == expected_changes
+
+
 def test_uninstalled(vim, chocolatey, chocolatey_mod):
     chocolatey.uninstalled(name="vim")
     result = chocolatey_mod.version(name="vim")

tests/pytests/functional/states/chocolatey/test_pre_20.py

@@ -109,6 +109,13 @@ def vim(chocolatey_mod):
     chocolatey_mod.uninstall(name="vim", force=True)
 
 
+@pytest.fixture(scope="function")
+def everything(chocolatey_mod):
+    chocolatey_mod.install(name="everything", version="1.4.1935")
+    yield
+    chocolatey_mod.uninstall(name="everything", force=True)
+
+
 def test_installed_latest(clean, chocolatey, chocolatey_mod):
     chocolatey.installed(name="vim")
     result = chocolatey_mod.version(name="vim")
@@ -122,6 +129,14 @@ def test_installed_version(clean, chocolatey, chocolatey_mod):
     assert result["vim"]["installed"][0] == "9.0.1672"
 
 
+def test_installed_version_existing_capitalization(
+    everything, chocolatey, chocolatey_mod
+):
+    result = chocolatey.installed(name="everything", version="1.4.11024")
+    expected_changes = {"Everything": {"new": ["1.4.11024"], "old": ["1.4.1935"]}}
+    assert result["changes"] == expected_changes
+
+
 def test_uninstalled(vim, chocolatey, chocolatey_mod):
     chocolatey.uninstalled(name="vim")
     result = chocolatey_mod.version(name="vim")

tests/pytests/functional/utils/gitfs/test_pillar.py

@@ -7,6 +7,7 @@ from salt.utils.gitfs import GitPillar, GitPython, Pygit2
 from salt.utils.immutabletypes import ImmutableDict, ImmutableList
 
 pytestmark = [
+    pytest.mark.windows_whitelisted,
     pytest.mark.slow_test,
 ]
 

tests/pytests/integration/netapi/test_ssh_client.py

@@ -192,6 +192,7 @@ def test_shell_inject_tgt(client, salt_ssh_roster_file, tmp_path, salt_auto_acco
         "eauth": "auto",
         "username": salt_auto_account.username,
         "password": salt_auto_account.password,
+        "ignore_host_keys": True,
     }
     ret = client.run(low)
     assert path.exists() is False

tests/pytests/unit/fileclient/test_fileclient.py

@@ -223,3 +223,19 @@ def test_get_file_client(file_client):
     with patch("salt.fileclient.RemoteClient", MagicMock(return_value="remote_client")):
         ret = fileclient.get_file_client(minion_opts)
         assert "remote_client" == ret
+
+
+def test_get_url_with_hash(client_opts):
+    """
+    Test get_url function with a URL containing a hash character.
+    """
+    with patch("os.path.isfile", return_value=True):
+        with patch("os.makedirs", return_value=None):
+            with patch("urllib.request.urlretrieve", return_value=None):
+                client = fileclient.Client(client_opts)
+                url = "file:///path/to/file#with#hash"
+                dest = "/mocked/destination"
+
+                result = client.get_url(url, dest)
+
+                assert result == "/path/to/file#with#hash"

tests/pytests/unit/modules/test_chocolatey.py

@@ -204,6 +204,20 @@ def test_version_check_remote_true():
         assert result == expected
 
 
+def test_version_check_remote_true_capitalization():
+    """
+    Test version when remote is True
+    """
+    list_side_effect = [
+        {"Ack": ["3.1.1"]},
+        {"Ack": ["3.1.1"], "Wolfpack": ["3.0.17"], "blackbird": ["1.0.79.3"]},
+    ]
+    with patch.object(chocolatey, "list_", side_effect=list_side_effect):
+        expected = {"ack": {"available": ["3.1.1"], "installed": ["3.1.1"]}}
+        result = chocolatey.version("ack", check_remote=True)
+        assert result == expected
+
+
 def test_version_check_remote_true_not_available():
     """
     Test version when remote is True but remote version is unavailable

tests/pytests/unit/pillar/test_pillar.py

@@ -164,6 +164,34 @@ def test_pillar_get_cache_disk(temp_salt_minion, caplog):
         assert fresh_pillar == {}
 
 
+def test_pillar_fetch_pillar_override_skipped(temp_salt_minion, caplog):
+    with pytest.helpers.temp_directory() as temp_path:
+        tmp_cachedir = Path(str(temp_path) + "/pillar_cache/")
+        tmp_cachedir.mkdir(parents=True)
+        assert tmp_cachedir.exists()
+        tmp_cachefile = Path(str(temp_path) + "/pillar_cache/" + temp_salt_minion.id)
+        assert tmp_cachefile.exists() is False
+
+        opts = temp_salt_minion.config.copy()
+        opts["pillarenv"] = None
+        opts["pillar_cache"] = True
+        opts["cachedir"] = str(temp_path)
+
+        pillar_override = {"inline_pillar": True}
+
+        caplog.at_level(logging.DEBUG)
+        pillar = salt.pillar.PillarCache(
+            opts=opts,
+            grains=salt.loader.grains(opts),
+            minion_id=temp_salt_minion.id,
+            saltenv="base",
+            pillar_override=pillar_override,
+        )
+
+        fresh_pillar = pillar.fetch_pillar()
+        assert fresh_pillar == {}
+
+
 def test_remote_pillar_timeout(temp_salt_minion, tmp_path):
     opts = temp_salt_minion.config.copy()
     opts["master_uri"] = "tcp://127.0.0.1:12323"