[3006.x] saltutil.refresh_grains(clean_pillar_cache=False) (#64176)

* flacky jail most have label

* add clean_pillar_cache arg

* add docs

* Revert "flacky jail most have label"

This reverts commit 0888b84fd0.

* add clean pillar cache refresh util

* dont clear unless refresh_pillar

* tests

* changelong

changelog/64081.fixed.md

@@ -0,0 +1 @@
+Fixed clear pillar cache on every highstate and added clean_pillar_cache=False to saltutil functions.

conf/master

@@ -1025,6 +1025,7 @@
 # If and only if a master has set ``pillar_cache: True``, the cache TTL controls the amount
 # of time, in seconds, before the cache is considered invalid by a master and a fresh
 # pillar is recompiled and stored.
+# The cache TTL does not prevent pillar cache from being refreshed before its TTL expires.
 #pillar_cache_ttl: 3600
 
 # If and only if a master has set `pillar_cache: True`, one of several storage providers

conf/suse/master

@@ -950,6 +950,7 @@ syndic_user: salt
 # If and only if a master has set ``pillar_cache: True``, the cache TTL controls the amount
 # of time, in seconds, before the cache is considered invalid by a master and a fresh
 # pillar is recompiled and stored.
+# The cache TTL does not prevent pillar cache from being refreshed before its TTL expires.
 #pillar_cache_ttl: 3600
 
 # If and only if a master has set `pillar_cache: True`, one of several storage providers

doc/ref/configuration/master.rst

@@ -5013,6 +5013,7 @@ Default: ``3600``
 If and only if a master has set ``pillar_cache: True``, the cache TTL controls the amount
 of time, in seconds, before the cache is considered invalid by a master and a fresh
 pillar is recompiled and stored.
+The cache TTL does not prevent pillar cache from being refreshed before its TTL expires.
 
 .. conf_master:: pillar_cache_backend
 

pkg/common/conf/master

@@ -1025,6 +1025,7 @@ user: salt
 # If and only if a master has set ``pillar_cache: True``, the cache TTL controls the amount
 # of time, in seconds, before the cache is considered invalid by a master and a fresh
 # pillar is recompiled and stored.
+# The cache TTL does not prevent pillar cache from being refreshed before its TTL expires.
 #pillar_cache_ttl: 3600
 
 # If and only if a master has set `pillar_cache: True`, one of several storage providers

salt/modules/saltutil.py

@@ -381,6 +381,9 @@ def refresh_grains(**kwargs):
     refresh_pillar : True
         Set to ``False`` to keep pillar data from being refreshed.
 
+    clean_pillar_cache : False
+        Set to ``True`` to refresh pillar cache.
+
     CLI Examples:
 
     .. code-block:: bash
@@ -389,6 +392,7 @@ def refresh_grains(**kwargs):
     """
     kwargs = salt.utils.args.clean_kwargs(**kwargs)
     _refresh_pillar = kwargs.pop("refresh_pillar", True)
+    clean_pillar_cache = kwargs.pop("clean_pillar_cache", False)
     if kwargs:
         salt.utils.args.invalid_kwargs(kwargs)
     # Modules and pillar need to be refreshed in case grains changes affected
@@ -396,14 +400,18 @@ def refresh_grains(**kwargs):
     # newly-reloaded grains to each execution module's __grains__ dunder.
     if _refresh_pillar:
         # we don't need to call refresh_modules here because it's done by refresh_pillar
-        refresh_pillar()
+        refresh_pillar(clean_cache=clean_pillar_cache)
     else:
         refresh_modules()
     return True
 
 
 def sync_grains(
-    saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist=None
+    saltenv=None,
+    refresh=True,
+    extmod_whitelist=None,
+    extmod_blacklist=None,
+    clean_pillar_cache=False,
 ):
     """
     .. versionadded:: 0.10.0
@@ -430,6 +438,9 @@ def sync_grains(
     extmod_blacklist : None
         comma-separated list of modules to blacklist based on type
 
+    clean_pillar_cache : False
+        Set to ``True`` to refresh pillar cache.
+
     CLI Examples:
 
     .. code-block:: bash
@@ -441,7 +452,7 @@ def sync_grains(
     ret = _sync("grains", saltenv, extmod_whitelist, extmod_blacklist)
     if refresh:
         # we don't need to call refresh_modules here because it's done by refresh_pillar
-        refresh_pillar()
+        refresh_pillar(clean_cache=clean_pillar_cache)
     return ret
 
 
@@ -915,7 +926,11 @@ def sync_log_handlers(
 
 
 def sync_pillar(
-    saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist=None
+    saltenv=None,
+    refresh=True,
+    extmod_whitelist=None,
+    extmod_blacklist=None,
+    clean_pillar_cache=False,
 ):
     """
     .. versionadded:: 2015.8.11,2016.3.2
@@ -935,6 +950,9 @@ def sync_pillar(
     extmod_blacklist : None
         comma-separated list of modules to blacklist based on type
 
+    clean_pillar_cache : False
+        Set to ``True`` to refresh pillar cache.
+
     .. note::
         This function will raise an error if executed on a traditional (i.e.
         not masterless) minion
@@ -953,7 +971,7 @@ def sync_pillar(
     ret = _sync("pillar", saltenv, extmod_whitelist, extmod_blacklist)
     if refresh:
         # we don't need to call refresh_modules here because it's done by refresh_pillar
-        refresh_pillar()
+        refresh_pillar(clean_cache=clean_pillar_cache)
     return ret
 
 
@@ -998,7 +1016,13 @@ def sync_executors(
     return ret
 
 
-def sync_all(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist=None):
+def sync_all(
+    saltenv=None,
+    refresh=True,
+    extmod_whitelist=None,
+    extmod_blacklist=None,
+    clean_pillar_cache=False,
+):
     """
     .. versionchanged:: 2015.8.11,2016.3.2
         On masterless minions, pillar modules are now synced, and refreshed
@@ -1036,6 +1060,9 @@ def sync_all(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist
     extmod_blacklist : None
         dictionary of modules to blacklist based on type
 
+    clean_pillar_cache : False
+        Set to ``True`` to refresh pillar cache.
+
     CLI Examples:
 
     .. code-block:: bash
@@ -1080,7 +1107,7 @@ def sync_all(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist
         ret["pillar"] = sync_pillar(saltenv, False, extmod_whitelist, extmod_blacklist)
     if refresh:
         # we don't need to call refresh_modules here because it's done by refresh_pillar
-        refresh_pillar()
+        refresh_pillar(clean_cache=clean_pillar_cache)
     return ret
 
 

tests/pytests/unit/modules/test_saltutil.py

@@ -2,13 +2,13 @@ import pytest
 
 import salt.modules.saltutil as saltutil
 from salt.client import LocalClient
-from tests.support.mock import create_autospec
+from tests.support.mock import create_autospec, patch
 from tests.support.mock import sentinel as s
 
 
 @pytest.fixture
 def configure_loader_modules():
-    return {saltutil: {}}
+    return {saltutil: {"__opts__": {"file_client": "local"}}}
 
 
 def test_exec_kwargs():
@@ -82,3 +82,57 @@ def test_exec_kwargs():
         **{"subset": s.subset, "batch": s.batch}
     )
     client.cmd_batch.assert_called_with(batch=s.batch, **_cmd_expected_kwargs)
+
+
+def test_refresh_grains_default_clean_pillar_cache():
+    with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar:
+        saltutil.refresh_grains()
+        refresh_pillar.assert_called_with(clean_cache=False)
+
+
+def test_refresh_grains_clean_pillar_cache():
+    with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar:
+        saltutil.refresh_grains(clean_pillar_cache=True)
+        refresh_pillar.assert_called_with(clean_cache=True)
+
+
+def test_sync_grains_default_clean_pillar_cache():
+    with patch("salt.modules.saltutil._sync"):
+        with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar:
+            saltutil.sync_grains()
+            refresh_pillar.assert_called_with(clean_cache=False)
+
+
+def test_sync_grains_clean_pillar_cache():
+    with patch("salt.modules.saltutil._sync"):
+        with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar:
+            saltutil.sync_grains(clean_pillar_cache=True)
+            refresh_pillar.assert_called_with(clean_cache=True)
+
+
+def test_sync_pillar_default_clean_pillar_cache():
+    with patch("salt.modules.saltutil._sync"):
+        with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar:
+            saltutil.sync_pillar()
+            refresh_pillar.assert_called_with(clean_cache=False)
+
+
+def test_sync_pillar_clean_pillar_cache():
+    with patch("salt.modules.saltutil._sync"):
+        with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar:
+            saltutil.sync_pillar(clean_pillar_cache=True)
+            refresh_pillar.assert_called_with(clean_cache=True)
+
+
+def test_sync_all_default_clean_pillar_cache():
+    with patch("salt.modules.saltutil._sync"):
+        with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar:
+            saltutil.sync_all()
+            refresh_pillar.assert_called_with(clean_cache=False)
+
+
+def test_sync_all_clean_pillar_cache():
+    with patch("salt.modules.saltutil._sync"):
+        with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar:
+            saltutil.sync_all(clean_pillar_cache=True)
+            refresh_pillar.assert_called_with(clean_cache=True)