Pillar cache for master

Entry point for pillar cache

Working msgpack implementation along with in-memory

Remove debugging

Perms for pillar cache creation

Documentation and small typo fix

Additional documentation note in master config

A note on pillar caching in the documentation on scaling

Backport of pillar cache to 2015.8 branch

Fixed ext pillar with debug

Lint

conf/master

@@ -617,6 +617,43 @@
 # Recursively merge lists by aggregating them instead of replacing them.
 #pillar_merge_lists: False
 
+# A master can cache pillars locally to bypass the expense of having to render them
+# for each minion on every request. This feature should only be enabled in cases
+# where pillar rendering time is known to be unsatisfactory and any attendent security
+# concerns about storing pillars in a master cache have been addressed.
+#
+# When enabling this feature, be certain to read through the additional pillar_cache_*
+# configuration options to fully understand the tuneable parameters and their implications.
+#
+#pillar_cache: False
+
+# If and only if a master has set `pillar_cache: True`, the cache TTL controls the amount
+# of time, in seconds, before the cache is considered invalid by a master and a fresh
+# pillar is recompiled and stored.
+#
+# pillar_cache_ttl: 3600
+
+# If an only if a master has set `pillar_cache: True`, one of several storage providers
+# can be utililzed.
+#
+# `disk`: The default storage backend. This caches rendered pillars to the master cache.
+#         Rendered pillars are serialized and deserialized as msgpack structures for speed.
+#         Note that pillars are stored UNENCRYPTED. Ensure that the master cache
+#         has permissions set appropriately. (Sane defaults are provided.)
+#
+#`memory`: [EXPERIMENTAL] An optional backend for pillar caches which uses a pure-Python
+#          in-memory data structure for maximal performance. There are several cavaets,
+#          however. First, because each master worker contains its own in-memory cache,
+#          there is no guarantee of cache consistency between minion requests. This
+#          works best in situations where the pillar rarely if ever changes. Secondly,
+#          and perhaps more importantly, this means that unencrypted pillars will
+#          be accessible to any process which can examine the memory of the salt-master!
+#          This may represent a substantial security risk.
+#
+#pillar_cache_backend: disk
+
+
+
 
 #####          Syndic settings       #####
 ##########################################

doc/topics/tutorials/intro_scale.rst

@@ -214,6 +214,23 @@ influence the key-size can have.
 Downsizing the Salt Master's key is not that important, because the minions
 do not encrypt as many messages as the Master does.
 
+In installations with large or with complex pillar files, it is possible
+for the master to exhibit poor performance as a result of having to render
+many pillar files at once. This exhibit itself in a number of ways, both
+as high load on the master and on minions which block on waiting for their
+pillar to be delivered to them.
+
+To reduce pillar rendering times, it is possible to cache pillars on the
+master. To do this, see the set of master configuration options which
+are prefixed with `pillar_cache`.
+
+.. note::
+
+    Caching pillars on the master may introduce security considerations.
+    Be certain to read caveats outlined in the master configuration file
+    to understand how pillar caching may affect a master's ability to
+    protect sensitive data!
+
 The Master is disk IO bound
 ---------------------------
 

salt/config.py

@@ -499,6 +499,15 @@ VALID_OPTS = {
     # Whether or not a copy of the master opts dict should be rendered into minion pillars
     'pillar_opts': bool,
 
+    # Cache the master pillar to disk to avoid having to pass through the rendering system
+    'pillar_cache': bool,
+
+    # Pillar cache TTL, in seconds. Has no effect unless `pillar_cache` is True
+    'pillar_cache_ttl': int,
+
+    # Pillar cache backend. Defaults to `disk` which stores caches in the master cache
+    'pillar_cache_backend': str,
+
     'pillar_safe_render_error': bool,
 
     # When creating a pillar, there are several strategies to choose from when
@@ -800,6 +809,11 @@ DEFAULT_MINION_OPTS = {
     'autoload_dynamic_modules': True,
     'environment': None,
     'pillarenv': None,
+    # `pillar_cache` and `pillar_ttl`
+    # are not used on the minion but are unavoidably in the code path
+    'pillar_cache': False,
+    'pillar_cache_ttl': 3600,
+    'pillar_cache_backend': 'disk',
     'extension_modules': '',
     'state_top': 'top.sls',
     'state_top_saltenv': None,
@@ -1060,6 +1074,9 @@ DEFAULT_MASTER_OPTS = {
     'pillar_safe_render_error': True,
     'pillar_source_merging_strategy': 'smart',
     'pillar_merge_lists': False,
+    'pillar_cache': False,
+    'pillar_cache_ttl': 3600,
+    'pillar_cache_backend': 'disk',
     'ping_on_rotate': False,
     'peer': {},
     'preserve_minion_cache': False,

salt/daemons/masterapi.py

@@ -700,7 +700,9 @@ class RemoteFuncs(object):
         '''
         if any(key not in load for key in ('id', 'grains')):
             return False
-        pillar = salt.pillar.Pillar(
+#        pillar = salt.pillar.Pillar(
+        log.debug('Master _pillar using ext: {0}'.format(load.get('ext')))
+        pillar = salt.pillar.get_pillar(
                 self.opts,
                 load['grains'],
                 load['id'],

salt/master.py

@@ -410,6 +410,15 @@ class Master(SMaster):
         if not self.opts['fileserver_backend']:
             errors.append('No fileserver backends are configured')
 
+        # Check to see if we need to create a pillar cache dir
+        if self.opts['pillar_cache'] and not os.path.isdir(os.path.join(self.opts['cachedir'], 'pillar_cache')):
+            try:
+                prev_umask = os.umask(0o077)
+                os.mkdir(os.path.join(self.opts['cachedir'], 'pillar_cache'))
+                os.umask(prev_umask)
+            except OSError:
+                pass
+
         non_legacy_git_pillars = [
             x for x in self.opts.get('ext_pillar', [])
             if 'git' in x
@@ -1126,7 +1135,8 @@ class AESFuncs(object):
         load['grains']['id'] = load['id']
 
         pillar_dirs = {}
-        pillar = salt.pillar.Pillar(
+#        pillar = salt.pillar.Pillar(
+        pillar = salt.pillar.get_pillar(
             self.opts,
             load['grains'],
             load['id'],

salt/pillar/__init__.py

@@ -9,6 +9,7 @@ import copy
 import os
 import collections
 import logging
+import tornado.gen
 
 # Import salt libs
 import salt.loader
@@ -17,6 +18,7 @@ import salt.minion
 import salt.crypt
 import salt.transport
 import salt.utils.url
+import salt.utils.cache
 from salt.exceptions import SaltClientError
 from salt.template import compile_template
 from salt.utils.dictupdate import merge
@@ -26,8 +28,6 @@ from salt.version import __version__
 # Import 3rd-party libs
 import salt.ext.six as six
 
-import tornado.gen
-
 log = logging.getLogger(__name__)
 
 
@@ -48,6 +48,13 @@ def get_pillar(opts, grains, id_, saltenv=None, ext=None, env=None, funcs=None,
         'remote': RemotePillar,
         'local': Pillar
     }.get(opts['file_client'], Pillar)
+    # If local pillar and we're caching, run through the cache system first
+    log.info('Determining pillar cache')
+    if opts['pillar_cache']:
+        log.info('Compiling pillar from cache')
+        log.debug('get_pillar using pillar cache with ext: {0}'.format(ext))
+        return PillarCache(opts, grains, id_, saltenv, ext=ext, functions=funcs,
+                pillar=pillar, pillarenv=pillarenv)
     return ptype(opts, grains, id_, saltenv, ext, functions=funcs,
                  pillar=pillar, pillarenv=pillarenv)
 
@@ -172,6 +179,95 @@ class RemotePillar(object):
         return ret_pillar
 
 
+class PillarCache(object):
+    '''
+    Return a cached pillar if it exists, otherwise cache it.
+
+    Pillar caches are structed in two diminensions: minion_id with a dict of saltenvs.
+    Each saltenv contains a pillar dict
+
+    Example data structure:
+
+    ```
+    {'minion_1':
+        {'base': {'pilar_key_1' 'pillar_val_1'}
+    }
+    '''
+    # TODO ABC?
+    def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None,
+            pillar=None, pillarenv=None):
+        # Yes, we need all of these because we need to route to the Pillar object
+        # if we have no cache. This is another refactor target.
+
+        # Go ahead and assign these because they may be needed later
+        self.opts = opts
+        self.grains = grains
+        self.minion_id = minion_id
+        self.ext = ext
+        self.functions = functions
+        self.pillar = pillar
+        self.pillarenv = pillarenv
+
+        if saltenv is None:
+            self.saltenv = 'base'
+        else:
+            self.saltenv = saltenv
+
+        # Determine caching backend
+        self.cache = salt.utils.cache.CacheFactory.factory(
+                self.opts['pillar_cache_backend'],
+                self.opts['pillar_cache_ttl'],
+                minion_cache_path=self._minion_cache_path(minion_id))
+
+    def _minion_cache_path(self, minion_id):
+        '''
+        Return the path to the cache file for the minion.
+
+        Used only for disk-based backends
+        '''
+        return os.path.join(self.opts['cachedir'], 'pillar_cache', minion_id)
+
+    def fetch_pillar(self):
+        '''
+        In the event of a cache miss, we need to incur the overhead of caching
+        a new pillar.
+        '''
+        log.debug('Pillar cache getting external pillar with ext: {0}'.format(self.ext))
+        fresh_pillar = Pillar(self.opts,
+                                 self.grains,
+                                 self.minion_id,
+                                 self.saltenv,
+                                 ext=self.ext,
+                                 functions=self.functions,
+                                 pillar=self.pillar,
+                                 pillarenv=self.pillarenv)
+        return fresh_pillar.compile_pillar()  # FIXME We are not yet passing pillar_dirs in here
+
+    def compile_pillar(self, *args, **kwargs):  # Will likely just be pillar_dirs
+        log.debug('Scanning pillar cache for information about minion {0} and saltenv {1}'.format(self.minion_id, self.saltenv))
+        log.debug('Scanning cache: {0}'.format(self.cache._dict))
+        # Check the cache!
+        if self.minion_id in self.cache:  # Keyed by minion_id
+            # TODO Compare grains, etc?
+            if self.saltenv in self.cache[self.minion_id]:
+                # We have a cache hit! Send it back.
+                log.debug('Pillar cache hit for minion {0} and saltenv {1}'.format(self.minion_id, self.saltenv))
+                return self.cache[self.minion_id][self.saltenv]
+            else:
+                # We found the minion but not the env. Store it.
+                fresh_pillar = self.fetch_pillar()
+                self.cache[self.minion_id][self.saltenv] = fresh_pillar
+                log.debug('Pillar cache miss for saltenv {0} for minion {1}'.format(self.saltenv, self.minion_id))
+                return fresh_pillar
+        else:
+            # We haven't seen this minion yet in the cache. Store it.
+            fresh_pillar = self.fetch_pillar()
+            self.cache[self.minion_id] = {self.saltenv: fresh_pillar}
+            log.debug('Pillar cache miss for minion {0}'.format(self.minion_id))
+            log.debug('Current pillar cache: {0}'.format(self.cache._dict))  # FIXME hack!
+            return fresh_pillar
+
+
 class Pillar(object):
     '''
     Read over the pillar top files and render the pillar data

salt/utils/cache.py

@@ -4,6 +4,8 @@ from __future__ import absolute_import, print_function
 import os
 import re
 import time
+import logging
+import msgpack
 
 # Import salt libs
 import salt.config
@@ -18,6 +20,23 @@ try:
 except ImportError:
     HAS_ZMQ = False
 
+log = logging.getLogger(__name__)
+
+
+class CacheFactory(object):
+    '''
+    Cache which can use a number of backends
+    '''
+    @classmethod
+    def factory(cls, backend, ttl, *args, **kwargs):
+        log.info('Factory backend: {0}'.format(backend))
+        if backend == 'memory':
+            return CacheDict(ttl, *args, **kwargs)
+        elif backend == 'disk':
+            return CacheDisk(ttl, kwargs['minion_cache_path'], *args, **kwargs)
+        else:
+            log.error('CacheFactory received unrecognized cache type')
+
 
 class CacheDict(dict):
     '''
@@ -57,6 +76,58 @@ class CacheDict(dict):
         return dict.__contains__(self, key)
 
 
+class CacheDisk(CacheDict):
+    '''
+    Class that represents itself as a dictionary to a consumer
+    but uses a disk-based backend. Serialization and de-serialization
+    is done with msgpack
+    '''
+    def __init__(self, ttl, path, *args, **kwargs):
+        super(CacheDisk, self).__init__(ttl, *args, **kwargs)
+        self._path = path
+        self._dict = self._read()
+
+    def __contains__(self, key):
+        self._enforce_ttl_key(key)
+        return self._dict.__contains__(key)
+
+    def __getitem__(self, key):
+        '''
+        Check if the key is ttld out, then do the get
+        '''
+        self._enforce_ttl_key(key)
+        return self._dict.__getitem__(key)
+
+    def __setitem__(self, key, val):
+        '''
+        Make sure to update the key cache time
+        '''
+        self._key_cache_time[key] = time.time()
+        self._dict.__setitem__(key, val)
+        # Do the same as the parent but also persist
+        self._write()
+
+    def _read(self):
+        '''
+        Read in from disk
+        '''
+        if not os.path.exists(self._path):
+            return {}
+        with salt.utils.fopen(self._path, 'r') as fp_:
+            cache = msgpack.load(fp_)
+        log.debug('Disk cache retreive: {0}'.format(cache))
+        return cache
+
+    def _write(self):
+        '''
+        Write out to disk
+        '''
+        # TODO Add check into preflight to ensure dir exists
+        # TODO Dir hashing?
+        with salt.utils.fopen(self._path, 'w+') as fp_:
+            msgpack.dump(self._dict, fp_)
+
+
 class CacheCli(object):
     '''
     Connection client for the ConCache. Should be used by all