From ab08fcb1375688c311127686a30fcd38f5098ddb Mon Sep 17 00:00:00 2001 From: Mike Place Date: Sun, 24 Jul 2016 10:31:18 -0600 Subject: [PATCH 1/8] Config and cache write implemented --- salt/config/__init__.py | 7 +++++++ salt/master.py | 24 ++++++++++++++++++++++++ salt/utils/minions.py | 6 ++++++ 3 files changed, 37 insertions(+) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index d6163cc31e2..c5e96213a58 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -159,6 +159,12 @@ VALID_OPTS = { # master 'syndic_finger': str, + # The caching mechanism to use for the PKI key store. Can substantially decrease master publish + # times. Available types: + # 'maint': Runs on a schedule as a part of the maintanence process. + # '': Disable the key cache [default] + 'key_cache': str, + # The user under which the daemon should run 'user': str, @@ -1134,6 +1140,7 @@ DEFAULT_MASTER_OPTS = { 'keep_jobs': 24, 'root_dir': salt.syspaths.ROOT_DIR, 'pki_dir': os.path.join(salt.syspaths.CONFIG_DIR, 'pki', 'master'), + 'key_cache': '', 'cachedir': os.path.join(salt.syspaths.CACHE_DIR, 'master'), 'file_roots': { 'base': [salt.syspaths.BASE_FILE_ROOTS_DIR, diff --git a/salt/master.py b/salt/master.py index 2319b909777..418a439552a 100644 --- a/salt/master.py +++ b/salt/master.py @@ -157,6 +157,8 @@ class Maintenance(SignalHandlingMultiprocessingProcess): self.loop_interval = int(self.opts['loop_interval']) # Track key rotation intervals self.rotate = int(time.time()) + # A serializer for general maint operations + self.serial = salt.payload.Serial(self.opts) # __setstate__ and __getstate__ are only used on Windows. # We do this so that __init__ will be invoked on Windows in the child @@ -237,6 +239,7 @@ class Maintenance(SignalHandlingMultiprocessingProcess): self.handle_search(now, last) self.handle_git_pillar() self.handle_schedule() + self.handle_key_cache() self.handle_presence(old_present) self.handle_key_rotate(now) salt.daemons.masterapi.fileserver_update(self.fileserver) @@ -251,6 +254,27 @@ class Maintenance(SignalHandlingMultiprocessingProcess): if self.opts.get('search'): if now - last >= self.opts['search_index_interval']: self.search.index() + + def handle_key_cache(self): + ''' + Evaluate accepted keys and create a msgpack file + which contains a list + ''' + if self.opts['key_cache'] == 'maint': + keys = [] + #TODO DRY from CKMinions + if self.opts['transport'] in ('zeromq', 'tcp'): + acc = 'minions' + else: + acc = 'accepted' + + for fn_ in os.listdir(os.path.join(self.opts['pki_dir'], acc)): + if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], acc, fn_)): + keys.append(fn_) + log.debug('Writing master key cache') + # Write a temporary file securely + with salt.utils.atomicfile.atomic_open(os.path.join(self.opts['pki_dir'], acc, '.key_cache')) as cache_file: + self.serial.dump(keys, cache_file) def handle_key_rotate(self, now): ''' diff --git a/salt/utils/minions.py b/salt/utils/minions.py index b01a30b25e8..7920b76f8be 100644 --- a/salt/utils/minions.py +++ b/salt/utils/minions.py @@ -221,6 +221,12 @@ class CkMinions(object): except OSError: return [] + def _pki_cache_minions(self): + ''' + Retreive complete minion list from PKI cache + ''' + pki_cache_fn = os.path.join(self.opts['pki_dir'], self.acc) + def _check_cache_minions(self, expr, delimiter, From 99c7a38f1c7829cf3df08c6d095ebe61ef52a476 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Sun, 24 Jul 2016 11:37:32 -0600 Subject: [PATCH 2/8] Beginning matcher func migration --- salt/utils/minions.py | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/salt/utils/minions.py b/salt/utils/minions.py index 7920b76f8be..f5062e8e84d 100644 --- a/salt/utils/minions.py +++ b/salt/utils/minions.py @@ -185,15 +185,7 @@ class CkMinions(object): ''' Return the minions found by looking via globs ''' - pki_dir = os.path.join(self.opts['pki_dir'], self.acc) - try: - files = [] - for fn_ in salt.utils.isorted(os.listdir(pki_dir)): - if not fn_.startswith('.') and os.path.isfile(os.path.join(pki_dir, fn_)): - files.append(fn_) - return fnmatch.filter(files, expr) - except OSError: - return [] + return fnmatch.filter(self._pki_minions(), expr) def _check_list_minions(self, expr, greedy): # pylint: disable=unused-argument ''' @@ -202,10 +194,11 @@ class CkMinions(object): if isinstance(expr, six.string_types): expr = [m for m in expr.split(',') if m] ret = [] - for minion in expr: - if os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, minion)): - ret.append(minion) - return ret + return [x for x in expr if x in self._pki_minions()] +# for minion in expr: +# if os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, minion)): +# ret.append(minion) +# return ret def _check_pcre_minions(self, expr, greedy): # pylint: disable=unused-argument ''' @@ -221,11 +214,26 @@ class CkMinions(object): except OSError: return [] - def _pki_cache_minions(self): + def _pki_minions(self): ''' - Retreive complete minion list from PKI cache + Retreive complete minion list from PKI dir. + Respects cache if configured ''' - pki_cache_fn = os.path.join(self.opts['pki_dir'], self.acc) + minions = [] + pki_cache_fn = os.path.join(self.opts['pki_dir'], self.acc, '.key_cache') + try: + if self.opts['key_cache'] and os.path.exists(pki_cache_fn): + log.debug('Returning cached minion list') + with salt.utils.fopen(pki_cache_fn) as fn_: + return self.serial.load(fn_) + else: + for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], elf.acc))): + if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)): + minions.append(fn_) + return minions + except OSError as exc: + log.error('Encountered OSError while evaluating minions in PKI dir: {0}'.format(exc)) + return minions def _check_cache_minions(self, expr, From 75776d17ddc7fb150f610332297d4f69428701a7 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Sun, 24 Jul 2016 11:42:54 -0600 Subject: [PATCH 3/8] Finish func refactor --- salt/utils/minions.py | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/salt/utils/minions.py b/salt/utils/minions.py index f5062e8e84d..f6acfcf3bf9 100644 --- a/salt/utils/minions.py +++ b/salt/utils/minions.py @@ -193,26 +193,14 @@ class CkMinions(object): ''' if isinstance(expr, six.string_types): expr = [m for m in expr.split(',') if m] - ret = [] return [x for x in expr if x in self._pki_minions()] -# for minion in expr: -# if os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, minion)): -# ret.append(minion) -# return ret def _check_pcre_minions(self, expr, greedy): # pylint: disable=unused-argument ''' Return the minions found by looking via regular expressions ''' - try: - minions = [] - for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))): - if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)): - minions.append(fn_) - reg = re.compile(expr) - return [m for m in minions if reg.match(m)] - except OSError: - return [] + reg = re.compile(expr) + return [m for m in self._pki_minions if reg.match(m)] def _pki_minions(self): ''' @@ -330,11 +318,7 @@ class CkMinions(object): cache_enabled = self.opts.get('minion_data_cache', False) if greedy: - mlist = [] - for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))): - if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)): - mlist.append(fn_) - minions = set(mlist) + minions = set(self._pki_minions()) elif cache_enabled: minions = os.listdir(os.path.join(self.opts['cachedir'], 'minions')) else: @@ -436,11 +420,7 @@ class CkMinions(object): if not isinstance(expr, six.string_types) and not isinstance(expr, (list, tuple)): log.error('Compound target that is neither string, list nor tuple') return [] - mlist = [] - for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))): - if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)): - mlist.append(fn_) - minions = set(mlist) + minions = set(self._pki_minions()) log.debug('minions: {0}'.format(minions)) if self.opts.get('minion_data_cache', False): From 7a86af386698954b58965b85161315166cad0b0b Mon Sep 17 00:00:00 2001 From: Mike Place Date: Sun, 24 Jul 2016 11:45:43 -0600 Subject: [PATCH 4/8] Typos --- salt/utils/minions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/utils/minions.py b/salt/utils/minions.py index f6acfcf3bf9..4a6c14adc0b 100644 --- a/salt/utils/minions.py +++ b/salt/utils/minions.py @@ -200,7 +200,7 @@ class CkMinions(object): Return the minions found by looking via regular expressions ''' reg = re.compile(expr) - return [m for m in self._pki_minions if reg.match(m)] + return [m for m in self._pki_minions() if reg.match(m)] def _pki_minions(self): ''' @@ -215,7 +215,7 @@ class CkMinions(object): with salt.utils.fopen(pki_cache_fn) as fn_: return self.serial.load(fn_) else: - for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], elf.acc))): + for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))): if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)): minions.append(fn_) return minions From e793fe42033ab60aae4a644e4652a7e6650f812c Mon Sep 17 00:00:00 2001 From: Mike Place Date: Sun, 24 Jul 2016 11:51:24 -0600 Subject: [PATCH 5/8] Conf file and changed option name to 'sched' --- conf/master | 4 ++++ salt/master.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/conf/master b/conf/master index 93e401684a5..218b1e55b19 100644 --- a/conf/master +++ b/conf/master @@ -45,6 +45,10 @@ # Directory used to store public key data: #pki_dir: /etc/salt/pki/master +# Key cache. Increases master speed for large numbers of accepted +# keys. Available options: 'sched'. (Updates on a fixed schedule.) +#key_cache: '' + # Directory to store job and cache data: # This directory may contain sensitive data and should be protected accordingly. # diff --git a/salt/master.py b/salt/master.py index 418a439552a..522d322f12e 100644 --- a/salt/master.py +++ b/salt/master.py @@ -260,7 +260,7 @@ class Maintenance(SignalHandlingMultiprocessingProcess): Evaluate accepted keys and create a msgpack file which contains a list ''' - if self.opts['key_cache'] == 'maint': + if self.opts['key_cache'] == 'sched': keys = [] #TODO DRY from CKMinions if self.opts['transport'] in ('zeromq', 'tcp'): From 1ca13c006a7308f2c496c3c72517761778a7d10d Mon Sep 17 00:00:00 2001 From: Mike Place Date: Sun, 24 Jul 2016 11:54:21 -0600 Subject: [PATCH 6/8] Add note in scaling doc --- doc/topics/tutorials/intro_scale.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/topics/tutorials/intro_scale.rst b/doc/topics/tutorials/intro_scale.rst index 51b3e3f5a4f..3fef3ed3090 100644 --- a/doc/topics/tutorials/intro_scale.rst +++ b/doc/topics/tutorials/intro_scale.rst @@ -271,3 +271,17 @@ If the job cache is necessary there are (currently) 2 options: into a returner (not sent through the Master) - master_job_cache (New in `2014.7.0`): this will make the Master store the job data using a returner (instead of the local job cache on disk). + +If a master has many accepted keys, it may take a long time to publish a job +because the master much first determine the matching minions and deliver +that information back to the waiting client before the job can be published. + +To mitigate this, a key cache may be enabled. This will reduce the load +on the master to a single file open instead of thousands or tens of thousands. + +This cache is updated by the maintanence process, however, which means that +minions with keys that are accepted may not be targeted by the master +for up to sixty seconds by default. + +To enable the master key cache, set `key_cache: 'sched'` in the master +configuration file. From d19270c4376c3618a7b312f7459544373721817d Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 25 Jul 2016 08:53:29 -0600 Subject: [PATCH 7/8] Lint whitespace --- salt/master.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/master.py b/salt/master.py index 522d322f12e..65a4a25f618 100644 --- a/salt/master.py +++ b/salt/master.py @@ -254,7 +254,7 @@ class Maintenance(SignalHandlingMultiprocessingProcess): if self.opts.get('search'): if now - last >= self.opts['search_index_interval']: self.search.index() - + def handle_key_cache(self): ''' Evaluate accepted keys and create a msgpack file From 00ba482f70012ab7e29ad0da440fa373260c611f Mon Sep 17 00:00:00 2001 From: Mike Place Date: Wed, 3 Aug 2016 03:10:12 +0900 Subject: [PATCH 8/8] Note key lag in doc --- conf/master | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf/master b/conf/master index 218b1e55b19..97e3f4da3ef 100644 --- a/conf/master +++ b/conf/master @@ -47,6 +47,9 @@ # Key cache. Increases master speed for large numbers of accepted # keys. Available options: 'sched'. (Updates on a fixed schedule.) +# Note that enabling this feature means that minions will not be +# available to target for up to the length of the maintanence loop +# which by default is 60s. #key_cache: '' # Directory to store job and cache data: