From faee403d6282f9a75c0d1622b4043b1c8a5711fe Mon Sep 17 00:00:00 2001 From: Mathieu Parent Date: Tue, 31 Dec 2019 15:13:58 +0100 Subject: [PATCH] salt.renderers.gpg: Cache GPG data if gpg_cache=True --- conf/master | 18 ++++++++++++++++++ doc/topics/tutorials/intro_scale.rst | 12 ++++++++---- salt/config/__init__.py | 19 ++++++++++++++++--- salt/renderers/gpg.py | 20 ++++++++++++++++++++ 4 files changed, 62 insertions(+), 7 deletions(-) diff --git a/conf/master b/conf/master index 59133fd3c80..41a62f2f347 100644 --- a/conf/master +++ b/conf/master @@ -1024,6 +1024,24 @@ # #pillar_cache_backend: disk +# A master can also cache GPG data locally to bypass the expense of having to render them +# for each minion on every request. This feature should only be enabled in cases +# where pillar rendering time is known to be unsatisfactory and any attendant security +# concerns about storing decrypted GPG data in a master cache have been addressed. +# +# When enabling this feature, be certain to read through the additional ``gpg_cache_*`` +# configuration options to fully understand the tunable parameters and their implications. +#gpg_cache: False + +# If and only if a master has set ``gpg_cache: True``, the cache TTL controls the amount +# of time, in seconds, before the cache is considered invalid by a master and a fresh +# pillar is recompiled and stored. +#gpg_cache_ttl: 86400 + +# If and only if a master has set `gpg_cache: True`, one of several storage providers +# can be utilized. Available options are the same as ``pillar_cache_backend``. +#gpg_cache_backend: disk + ###### Reactor Settings ##### ########################################### diff --git a/doc/topics/tutorials/intro_scale.rst b/doc/topics/tutorials/intro_scale.rst index 009725f0bb9..80338fd7af1 100644 --- a/doc/topics/tutorials/intro_scale.rst +++ b/doc/topics/tutorials/intro_scale.rst @@ -227,12 +227,16 @@ To reduce pillar rendering times, it is possible to cache pillars on the master. To do this, see the set of master configuration options which are prefixed with `pillar_cache`. +If many pillars are encrypted using :mod:`gpg ` renderer, it +is possible to cache GPG data. To do this, see the set of master configuration +options which are prefixed with `gpg_cache`. + .. note:: - Caching pillars on the master may introduce security considerations. - Be certain to read caveats outlined in the master configuration file - to understand how pillar caching may affect a master's ability to - protect sensitive data! + Caching pillars or GPG data on the master may introduce security + considerations. Be certain to read caveats outlined in the master + configuration file to understand how pillar caching may affect a master's + ability to protect sensitive data! The Master is disk IO bound --------------------------- diff --git a/salt/config/__init__.py b/salt/config/__init__.py index f0845fe9a20..a37050eef3f 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -577,6 +577,12 @@ VALID_OPTS = immutabletypes.freeze( "pillar_cache_ttl": int, # Pillar cache backend. Defaults to `disk` which stores caches in the master cache "pillar_cache_backend": six.string_types, + # Cache the GPG data to avoid having to pass through the gpg renderer + "gpg_cache": bool, + # GPG data cache TTL, in seconds. Has no effect unless `gpg_cache` is True + "gpg_cache_ttl": int, + # GPG data cache backend. Defaults to `disk` which stores caches in the master cache + "gpg_cache_backend": six.string_types, "pillar_safe_render_error": bool, # When creating a pillar, there are several strategies to choose from when # encountering duplicate values @@ -981,11 +987,15 @@ DEFAULT_MINION_OPTS = immutabletypes.freeze( "pillar_source_merging_strategy": "smart", "pillar_merge_lists": False, "pillar_includes_override_sls": False, - # ``pillar_cache``, ``pillar_cache_ttl`` and ``pillar_cache_backend`` + # ``pillar_cache``, ``pillar_cache_ttl``, ``pillar_cache_backend``, + # ``gpg_cache``, ``gpg_cache_ttl`` and ``gpg_cache_backend`` # are not used on the minion but are unavoidably in the code path "pillar_cache": False, "pillar_cache_ttl": 3600, "pillar_cache_backend": "disk", + "gpg_cache": False, + "gpg_cache_ttl": 86400, + "gpg_cache_backend": "disk", "extension_modules": os.path.join(salt.syspaths.CACHE_DIR, "minion", "extmods"), "state_top": "top.sls", "state_top_saltenv": None, @@ -1343,6 +1353,9 @@ DEFAULT_MASTER_OPTS = immutabletypes.freeze( "pillar_cache": False, "pillar_cache_ttl": 3600, "pillar_cache_backend": "disk", + "gpg_cache": False, + "gpg_cache_ttl": 86400, + "gpg_cache_backend": "disk", "ping_on_rotate": False, "peer": {}, "preserve_minion_cache": False, @@ -3521,7 +3534,7 @@ def apply_minion_config( log.warning( "The 'saltenv' and 'environment' minion config options " "cannot both be used. Ignoring 'environment' in favor of " - "'saltenv'.", + "'saltenv'." ) # Set environment to saltenv in case someone's custom module is # refrencing __opts__['environment'] @@ -3739,7 +3752,7 @@ def apply_master_config(overrides=None, defaults=None): log.warning( "The 'saltenv' and 'environment' master config options " "cannot both be used. Ignoring 'environment' in favor of " - "'saltenv'.", + "'saltenv'." ) # Set environment to saltenv in case someone's custom runner is # refrencing __opts__['environment'] diff --git a/salt/renderers/gpg.py b/salt/renderers/gpg.py index 05686227fa8..d9513e611d0 100644 --- a/salt/renderers/gpg.py +++ b/salt/renderers/gpg.py @@ -281,6 +281,7 @@ from subprocess import PIPE, Popen import salt.syspaths # Import salt libs +import salt.utils.cache import salt.utils.path import salt.utils.stringio import salt.utils.stringutils @@ -297,6 +298,7 @@ GPG_CIPHERTEXT = re.compile( ), re.DOTALL, ) +GPG_CACHE = None def _get_gpg_exec(): @@ -330,6 +332,18 @@ def _get_key_dir(): return gpg_keydir +def _get_cache(): + global GPG_CACHE + if not GPG_CACHE: + cachedir = __opts__.get("cachedir") + GPG_CACHE = salt.utils.cache.CacheFactory.factory( + __opts__.get("gpg_cache_backend"), + __opts__.get("gpg_cache_ttl"), + minion_cache_path=os.path.join(cachedir, "gpg_cache"), + ) + return GPG_CACHE + + def _decrypt_ciphertext(cipher): """ Given a block of ciphertext as a string, and a gpg object, try to decrypt @@ -342,6 +356,10 @@ def _decrypt_ciphertext(cipher): # ciphertext is binary pass cipher = salt.utils.stringutils.to_bytes(cipher) + if __opts__.get("gpg_cache"): + cache = _get_cache() + if cipher in cache: + return cache[cipher] cmd = [ _get_gpg_exec(), "--homedir", @@ -357,6 +375,8 @@ def _decrypt_ciphertext(cipher): log.warning("Could not decrypt cipher %r, received: %r", cipher, decrypt_error) return cipher else: + if __opts__.get("gpg_cache"): + cache[cipher] = decrypted_data return decrypted_data