Merge pull request #40497 from DSRCorporation/features/39275_memcache

Memcache documentation and minor updates.

conf/master

@@ -133,6 +133,14 @@
 
 # Cache subsystem module to use for minion data cache.
 #cache: localfs
+# Enables a fast in-memory cache booster and sets the expiration time.
+#memcache_expire_seconds: 0
+# Set a memcache limit in items (bank + key) per cache storage (driver + driver_opts).
+#memcache_max_items: 1024
+# Each time a cache storage got full cleanup all the expired items not just the oldest one.
+#memcache_full_cleanup: False
+# Enable collecting the memcache stats and log it on `debug` log level.
+#memcache_debug: False
 
 # Store all returns in the given returner.
 # Setting this option requires that any returner-specific configuration also

doc/ref/configuration/master.rst

@@ -481,6 +481,75 @@ Cache subsystem module to use for minion data cache.
 
     cache: consul
 
+.. conf_master:: memcache_expire_seconds
+
+``memcache_expire_seconds``
+---------------------------
+
+Default: ``0``
+
+Memcache is an additional cache layer that keeps a limited amount of data
+fetched from the minion data cache for a limited period of time in memory that
+makes cache operations faster. It doesn't make much sence for the ``localfs``
+cache driver but helps for more complex drivers like ``consul``.
+
+This option sets the memcache items expiration time. By default is set to ``0``
+that disables the memcache.
+
+.. code-block:: yaml
+
+    memcache_expire_seconds: 30
+
+.. conf_master:: memcache_max_items
+
+``memcache_max_items``
+----------------------
+
+Default: ``1024``
+
+Set memcache limit in items that are bank-key pairs. I.e the list of
+minion_0/data, minion_0/mine, minion_1/data contains 3 items. This value depends
+on the count of minions usually targeted in your environment. The best one could
+be found by analyzing the cache log with ``memcache_debug`` enabled.
+
+.. code-block:: yaml
+
+    memcache_max_items: 1024
+
+.. conf_master:: memcache_full_cleanup
+
+``memcache_full_cleanup``
+-------------------------
+
+Default: ``False``
+
+If cache storage got full, i.e. the items count exceeds the
+``memcache_max_items`` value, memcache cleans up it's storage. If this option
+set to ``False`` memcache removes the only one oldest value from it's storage.
+If this set set to ``True`` memcache removes all the expired items and also
+removes the oldest one if there are no expired items.
+
+.. code-block:: yaml
+
+    memcache_full_cleanup: True
+
+.. conf_master:: memcache_debug
+
+``memcache_debug``
+------------------
+
+Default: ``False``
+
+Enable collecting the memcache stats and log it on `debug` log level. If enabled
+memcache collect information about how many ``fetch`` calls has been done and
+how many of them has been hit by memcache. Also it outputs the rate value that
+is the result of division of the first two values. This should help to choose
+right values for the expiration time and the cache size.
+
+.. code-block:: yaml
+
+    memcache_debug: True
+
 .. conf_master:: ext_job_cache
 
 ``ext_job_cache``

doc/topics/releases/2016.11.4.rst

@@ -11,3 +11,13 @@ AIX Fixes
 Added module execution support for user and group
 
 
+Minion Data Cache Fixes
+=======================
+
+Added Memcache booster for the minion data cache.
+Memcache is an additional cache layer that keeps a limited amount of data
+fetched from the minion data cache for a limited period of time in memory that
+makes cache operations faster. It doesn't make much sence for the ``localfs``
+cache driver but helps for more complex drivers like ``consul``.
+For more details see ``memcache_expire_seconds`` and other ``memcache_*``
+options in the master config reverence.

salt/cache/__init__.py

@@ -289,6 +289,8 @@ class MemCache(Cache):
             for key, data in list(storage.items()):
                 if data[0] + expire < now:
                     del storage[key]
+                else:
+                    break
 
     def _get_storage_id(self):
         fun = '{0}.storage_id'.format(self.driver)
@@ -315,7 +317,7 @@ class MemCache(Cache):
         if record is not None and record[0] + self.expire >= now:
             if self.debug:
                 self.hit += 1
-                log.trace('MemCache stats (call/hit/rate): '
+                log.debug('MemCache stats (call/hit/rate): '
                           '{0}/{1}/{2}'.format(self.call,
                                                self.hit,
                                                float(self.hit) / self.call))
@@ -326,6 +328,11 @@ class MemCache(Cache):
 
         # Have no value for the key or value is expired
         data = super(MemCache, self).fetch(bank, key)
+        if len(self.storage) >= self.max:
+            if self.cleanup:
+                MemCache.__cleanup(self.expire)
+            if len(self.storage) >= self.max:
+                self.storage.popitem(last=False)
         self.storage[(bank, key)] = [now, data]
         return data
 
@@ -335,7 +342,7 @@ class MemCache(Cache):
         if len(self.storage) >= self.max:
             if self.cleanup:
                 MemCache.__cleanup(self.expire)
-            else:
+            if len(self.storage) >= self.max:
                 self.storage.popitem(last=False)
         self.storage[(bank, key)] = [time.time(), data]
 

salt/config/__init__.py

@@ -921,11 +921,11 @@ VALID_OPTS = {
     'cache': str,
     # Enables a fast in-memory cache booster and sets the expiration time.
     'memcache_expire_seconds': int,
-    # Set a memcache limit in items per cache storage (driver+driver opts).
+    # Set a memcache limit in items (bank + key) per cache storage (driver + driver_opts).
     'memcache_max_items': int,
     # Each time a cache storage got full cleanup all the expired items not just the oldest one.
     'memcache_full_cleanup': bool,
-    # Enable collecting theh memcache stats and log it on `trace` log level.
+    # Enable collecting the memcache stats and log it on `debug` log level.
     'memcache_debug': bool,
 
     # Extra modules for Salt Thin