diff --git a/changelog/64081.fixed.md b/changelog/64081.fixed.md new file mode 100644 index 00000000000..ed0720ac9a3 --- /dev/null +++ b/changelog/64081.fixed.md @@ -0,0 +1 @@ +Fixed clear pillar cache on every highstate and added clean_pillar_cache=False to saltutil functions. diff --git a/conf/master b/conf/master index f542051d762..2c0a5c9cb87 100644 --- a/conf/master +++ b/conf/master @@ -1025,6 +1025,7 @@ # If and only if a master has set ``pillar_cache: True``, the cache TTL controls the amount # of time, in seconds, before the cache is considered invalid by a master and a fresh # pillar is recompiled and stored. +# The cache TTL does not prevent pillar cache from being refreshed before its TTL expires. #pillar_cache_ttl: 3600 # If and only if a master has set `pillar_cache: True`, one of several storage providers diff --git a/conf/suse/master b/conf/suse/master index 7168441dc41..863d8790240 100644 --- a/conf/suse/master +++ b/conf/suse/master @@ -950,6 +950,7 @@ syndic_user: salt # If and only if a master has set ``pillar_cache: True``, the cache TTL controls the amount # of time, in seconds, before the cache is considered invalid by a master and a fresh # pillar is recompiled and stored. +# The cache TTL does not prevent pillar cache from being refreshed before its TTL expires. #pillar_cache_ttl: 3600 # If and only if a master has set `pillar_cache: True`, one of several storage providers diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index a6022c94ee1..74d4b58b084 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -5013,6 +5013,7 @@ Default: ``3600`` If and only if a master has set ``pillar_cache: True``, the cache TTL controls the amount of time, in seconds, before the cache is considered invalid by a master and a fresh pillar is recompiled and stored. +The cache TTL does not prevent pillar cache from being refreshed before its TTL expires. .. conf_master:: pillar_cache_backend diff --git a/pkg/common/conf/master b/pkg/common/conf/master index fcad1961c10..4f0fa646d49 100644 --- a/pkg/common/conf/master +++ b/pkg/common/conf/master @@ -1025,6 +1025,7 @@ user: salt # If and only if a master has set ``pillar_cache: True``, the cache TTL controls the amount # of time, in seconds, before the cache is considered invalid by a master and a fresh # pillar is recompiled and stored. +# The cache TTL does not prevent pillar cache from being refreshed before its TTL expires. #pillar_cache_ttl: 3600 # If and only if a master has set `pillar_cache: True`, one of several storage providers diff --git a/salt/modules/saltutil.py b/salt/modules/saltutil.py index a692c3f34d4..ecf467046aa 100644 --- a/salt/modules/saltutil.py +++ b/salt/modules/saltutil.py @@ -381,6 +381,9 @@ def refresh_grains(**kwargs): refresh_pillar : True Set to ``False`` to keep pillar data from being refreshed. + clean_pillar_cache : False + Set to ``True`` to refresh pillar cache. + CLI Examples: .. code-block:: bash @@ -389,6 +392,7 @@ def refresh_grains(**kwargs): """ kwargs = salt.utils.args.clean_kwargs(**kwargs) _refresh_pillar = kwargs.pop("refresh_pillar", True) + clean_pillar_cache = kwargs.pop("clean_pillar_cache", False) if kwargs: salt.utils.args.invalid_kwargs(kwargs) # Modules and pillar need to be refreshed in case grains changes affected @@ -396,14 +400,18 @@ def refresh_grains(**kwargs): # newly-reloaded grains to each execution module's __grains__ dunder. if _refresh_pillar: # we don't need to call refresh_modules here because it's done by refresh_pillar - refresh_pillar() + refresh_pillar(clean_cache=clean_pillar_cache) else: refresh_modules() return True def sync_grains( - saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist=None + saltenv=None, + refresh=True, + extmod_whitelist=None, + extmod_blacklist=None, + clean_pillar_cache=False, ): """ .. versionadded:: 0.10.0 @@ -430,6 +438,9 @@ def sync_grains( extmod_blacklist : None comma-separated list of modules to blacklist based on type + clean_pillar_cache : False + Set to ``True`` to refresh pillar cache. + CLI Examples: .. code-block:: bash @@ -441,7 +452,7 @@ def sync_grains( ret = _sync("grains", saltenv, extmod_whitelist, extmod_blacklist) if refresh: # we don't need to call refresh_modules here because it's done by refresh_pillar - refresh_pillar() + refresh_pillar(clean_cache=clean_pillar_cache) return ret @@ -915,7 +926,11 @@ def sync_log_handlers( def sync_pillar( - saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist=None + saltenv=None, + refresh=True, + extmod_whitelist=None, + extmod_blacklist=None, + clean_pillar_cache=False, ): """ .. versionadded:: 2015.8.11,2016.3.2 @@ -935,6 +950,9 @@ def sync_pillar( extmod_blacklist : None comma-separated list of modules to blacklist based on type + clean_pillar_cache : False + Set to ``True`` to refresh pillar cache. + .. note:: This function will raise an error if executed on a traditional (i.e. not masterless) minion @@ -953,7 +971,7 @@ def sync_pillar( ret = _sync("pillar", saltenv, extmod_whitelist, extmod_blacklist) if refresh: # we don't need to call refresh_modules here because it's done by refresh_pillar - refresh_pillar() + refresh_pillar(clean_cache=clean_pillar_cache) return ret @@ -998,7 +1016,13 @@ def sync_executors( return ret -def sync_all(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist=None): +def sync_all( + saltenv=None, + refresh=True, + extmod_whitelist=None, + extmod_blacklist=None, + clean_pillar_cache=False, +): """ .. versionchanged:: 2015.8.11,2016.3.2 On masterless minions, pillar modules are now synced, and refreshed @@ -1036,6 +1060,9 @@ def sync_all(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist extmod_blacklist : None dictionary of modules to blacklist based on type + clean_pillar_cache : False + Set to ``True`` to refresh pillar cache. + CLI Examples: .. code-block:: bash @@ -1080,7 +1107,7 @@ def sync_all(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist ret["pillar"] = sync_pillar(saltenv, False, extmod_whitelist, extmod_blacklist) if refresh: # we don't need to call refresh_modules here because it's done by refresh_pillar - refresh_pillar() + refresh_pillar(clean_cache=clean_pillar_cache) return ret diff --git a/tests/pytests/unit/modules/test_saltutil.py b/tests/pytests/unit/modules/test_saltutil.py index 889543c9454..97527d3dc24 100644 --- a/tests/pytests/unit/modules/test_saltutil.py +++ b/tests/pytests/unit/modules/test_saltutil.py @@ -2,13 +2,13 @@ import pytest import salt.modules.saltutil as saltutil from salt.client import LocalClient -from tests.support.mock import create_autospec +from tests.support.mock import create_autospec, patch from tests.support.mock import sentinel as s @pytest.fixture def configure_loader_modules(): - return {saltutil: {}} + return {saltutil: {"__opts__": {"file_client": "local"}}} def test_exec_kwargs(): @@ -82,3 +82,57 @@ def test_exec_kwargs(): **{"subset": s.subset, "batch": s.batch} ) client.cmd_batch.assert_called_with(batch=s.batch, **_cmd_expected_kwargs) + + +def test_refresh_grains_default_clean_pillar_cache(): + with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar: + saltutil.refresh_grains() + refresh_pillar.assert_called_with(clean_cache=False) + + +def test_refresh_grains_clean_pillar_cache(): + with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar: + saltutil.refresh_grains(clean_pillar_cache=True) + refresh_pillar.assert_called_with(clean_cache=True) + + +def test_sync_grains_default_clean_pillar_cache(): + with patch("salt.modules.saltutil._sync"): + with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar: + saltutil.sync_grains() + refresh_pillar.assert_called_with(clean_cache=False) + + +def test_sync_grains_clean_pillar_cache(): + with patch("salt.modules.saltutil._sync"): + with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar: + saltutil.sync_grains(clean_pillar_cache=True) + refresh_pillar.assert_called_with(clean_cache=True) + + +def test_sync_pillar_default_clean_pillar_cache(): + with patch("salt.modules.saltutil._sync"): + with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar: + saltutil.sync_pillar() + refresh_pillar.assert_called_with(clean_cache=False) + + +def test_sync_pillar_clean_pillar_cache(): + with patch("salt.modules.saltutil._sync"): + with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar: + saltutil.sync_pillar(clean_pillar_cache=True) + refresh_pillar.assert_called_with(clean_cache=True) + + +def test_sync_all_default_clean_pillar_cache(): + with patch("salt.modules.saltutil._sync"): + with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar: + saltutil.sync_all() + refresh_pillar.assert_called_with(clean_cache=False) + + +def test_sync_all_clean_pillar_cache(): + with patch("salt.modules.saltutil._sync"): + with patch("salt.modules.saltutil.refresh_pillar") as refresh_pillar: + saltutil.sync_all(clean_pillar_cache=True) + refresh_pillar.assert_called_with(clean_cache=True)