Merge pull request #34009 from rallytime/bp-33948

Back-port #33948 to 2016.3 + add log message

doc/topics/releases/2016.3.2.rst

@@ -0,0 +1,13 @@
+===========================
+Salt 2016.3.2 Release Notes
+===========================
+
+Version 2016.3.2 is a bugfix release for :doc:`2016.3.0
+</topics/releases/2016.3.0>`.
+
+Returner Changes
+================
+
+  - Any returner which implements a ``save_load`` function is now required to
+    accept a ``minions`` keyword argument. All returners which ship with Salt
+    have been modified to do so.

salt/master.py

@@ -62,6 +62,7 @@ import salt.daemons.masterapi
 import salt.defaults.exitcodes
 import salt.transport.server
 import salt.log.setup
+import salt.utils.args
 import salt.utils.atomicfile
 import salt.utils.event
 import salt.utils.job
@@ -2259,21 +2260,40 @@ class ClearFuncs(object):
         self.event.fire_event(new_job_load, tagify([clear_load['jid'], 'new'], 'job'))
 
         if self.opts['ext_job_cache']:
+            fstr = '{0}.save_load'.format(self.opts['ext_job_cache'])
+            save_load_func = True
+
+            # Get the returner's save_load arg_spec.
             try:
-                fstr = '{0}.save_load'.format(self.opts['ext_job_cache'])
-                self.mminion.returners[fstr](clear_load['jid'], clear_load)
-            except KeyError:
+                arg_spec = salt.utils.args.get_function_argspec(fstr)
+
+                # Check if 'minions' is included in returner's save_load arg_spec.
+                # This may be missing in custom returners, which we should warn about.
+                if 'minions' not in arg_spec.args:
+                    log.critical(
+                        'The specified returner used for the external job cache '
+                        '\'{0}\' does not have a \'minions\' kwarg in the returner\'s '
+                        'save_load function.'.format(
+                            self.opts['ext_job_cache']
+                        )
+                    )
+            except AttributeError:
+                save_load_func = False
                 log.critical(
                     'The specified returner used for the external job cache '
                     '"{0}" does not have a save_load function!'.format(
                         self.opts['ext_job_cache']
                     )
                 )
-            except Exception:
-                log.critical(
-                    'The specified returner threw a stack trace:\n',
-                    exc_info=True
-                )
+
+            if save_load_func:
+                try:
+                    self.mminion.returners[fstr](clear_load['jid'], clear_load, minions=minions)
+                except Exception:
+                    log.critical(
+                        'The specified returner threw a stack trace:\n',
+                        exc_info=True
+                    )
 
         # always write out to the master job caches
         try:

salt/returners/cassandra_cql_return.py

@@ -236,7 +236,7 @@ def event_return(events):
             raise
 
 
-def save_load(jid, load):
+def save_load(jid, load, minions=None):
     '''
     Save the load to the specified jid id
     '''

salt/returners/couchbase_return.py

@@ -184,7 +184,7 @@ def returner(load):
         return False
 
 
-def save_load(jid, clear_load):
+def save_load(jid, clear_load, minion=None):
     '''
     Save the load to the specified jid
     '''

salt/returners/django_return.py

@@ -65,7 +65,7 @@ def returner(ret):
                   'which responded with {1}'.format(signal[0], signal[1]))
 
 
-def save_load(jid, load):
+def save_load(jid, load, minions=None):
     '''
     Save the load to the specified jid
     '''

salt/returners/elasticsearch_return.py

@@ -119,7 +119,7 @@ def prep_jid(nocache=False, passed_jid=None):  # pylint: disable=unused-argument
     return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid()
 
 
-def save_load(jid, load):
+def save_load(jid, load, minions=None):
     '''
     Save the load to the specified jid id
 

salt/returners/etcd_return.py

@@ -133,7 +133,7 @@ def returner(ret):
         client.set(dest, json.dumps(ret[field]), ttl=ttl)
 
 
-def save_load(jid, load):
+def save_load(jid, load, minions=None):
     '''
     Save the load to the specified jid
     '''

salt/returners/influxdb_return.py

@@ -137,7 +137,7 @@ def returner(ret):
         log.critical('Failed to store return with InfluxDB returner: {0}'.format(ex))
 
 
-def save_load(jid, load):
+def save_load(jid, load, minions=None):
     '''
     Save the load to the specified jid
     '''

salt/returners/memcache_return.py

@@ -154,7 +154,7 @@ def returner(ret):
     _append_list(serv, 'jids', jid)
 
 
-def save_load(jid, load):
+def save_load(jid, load, minions=None):
     '''
     Save the load to the specified jid
     '''

salt/returners/mongo_future_return.py

@@ -199,7 +199,7 @@ def returner(ret):
         mdb.saltReturns.insert(sdata.copy())
 
 
-def save_load(jid, load):
+def save_load(jid, load, minions=None):
     '''
     Save the load for a given job id
     '''

salt/returners/multi_returner.py

@@ -61,7 +61,7 @@ def returner(load):
         _mminion().returners['{0}.returner'.format(returner_)](load)
 
 
-def save_load(jid, clear_load):
+def save_load(jid, clear_load, minions=None):
     '''
     Write load to all returners in multi_returner
     '''

salt/returners/mysql.py

@@ -287,7 +287,7 @@ def event_return(events):
             cur.execute(sql, (tag, json.dumps(data), __opts__['id']))
 
 
-def save_load(jid, load):
+def save_load(jid, load, minions=None):
     '''
     Save the load to the specified jid id
     '''

salt/returners/odbc.py

@@ -214,7 +214,7 @@ def returner(ret):
     _close_conn(conn)
 
 
-def save_load(jid, load):
+def save_load(jid, load, minions=None):
     '''
     Save the load to the specified jid id
     '''

salt/returners/pgjsonb.py

@@ -272,7 +272,7 @@ def event_return(events):
                               __opts__['id'], time.strftime('%Y-%m-%d %H:%M:%S %z', time.localtime())))
 
 
-def save_load(jid, load):
+def save_load(jid, load, minions=None):
     '''
     Save the load to the specified jid id
     '''

salt/returners/postgres.py

@@ -189,7 +189,7 @@ def returner(ret):
     _close_conn(conn)
 
 
-def save_load(jid, load):
+def save_load(jid, load, minions=None):
     '''
     Save the load to the specified jid id
     '''

salt/returners/postgres_local_cache.py

@@ -264,7 +264,7 @@ def event_return(events):
     _close_conn(conn)
 
 
-def save_load(jid, clear_load):
+def save_load(jid, clear_load, minions=None):
     '''
     Save the load to the specified jid id
     '''

salt/returners/redis_return.py

@@ -120,7 +120,7 @@ def returner(ret):
     pipeline.execute()
 
 
-def save_load(jid, load):
+def save_load(jid, load, minions=None):
     '''
     Save the load to the specified jid
     '''

salt/returners/sqlite3_return.py

@@ -178,7 +178,7 @@ def returner(ret):
     _close_conn(conn)
 
 
-def save_load(jid, load):
+def save_load(jid, load, minions=None):
     '''
     Save the load to the specified jid
     '''

salt/utils/minions.py

@@ -654,13 +654,14 @@ class CkMinions(object):
 
         return set(self.check_minions(v_expr, v_matcher))
 
-    def validate_tgt(self, valid, expr, expr_form):
+    def validate_tgt(self, valid, expr, expr_form, minions=None):
         '''
         Return a Bool. This function returns if the expression sent in is
         within the scope of the valid expression
         '''
         v_minions = self._expand_matching(valid)
-        minions = set(self.check_minions(expr, expr_form))
+        if minions is None:
+            minions = set(self.check_minions(expr, expr_form))
         d_bool = not bool(minions.difference(v_minions))
         if len(v_minions) == len(minions) and d_bool:
             return True
@@ -818,7 +819,8 @@ class CkMinions(object):
                    tgt,
                    tgt_type='glob',
                    groups=None,
-                   publish_validate=False):
+                   publish_validate=False,
+                   minions=None):
         '''
         Returns a bool which defines if the requested function is authorized.
         Used to evaluate the standard structure under external master
@@ -860,7 +862,8 @@ class CkMinions(object):
                         if self.validate_tgt(
                             valid,
                             tgt,
-                            tgt_type):
+                            tgt_type,
+                            minions=minions):
                             # Minions are allowed, verify function in allowed list
                             if isinstance(ind[valid], six.string_types):
                                 if self.match_check(ind[valid], fun):