Merge pull request #38979 from rallytime/merge-2016.11

[2016.11] Merge forward from 2016.3 to 2016.11

conf/master

@@ -704,6 +704,11 @@
 # ext_pillar.
 #ext_pillar_first: False
 
+# The external pillars permitted to be used on-demand using pillar.ext
+#on_demand_ext_pillar:
+#  - libvirt
+#  - virtkey
+
 # The pillar_gitfs_ssl_verify option specifies whether to ignore ssl certificate
 # errors when contacting the pillar gitfs backend. You might want to set this to
 # false if you're using a git backend that uses a self-signed certificate but

doc/ref/configuration/master.rst

@@ -2517,6 +2517,34 @@ configuration is the same as :conf_master:`file_roots`:
       prod:
         - /srv/pillar/prod
 
+.. conf_master:: on_demand_ext_pillar
+
+``on_demand_ext_pillar``
+------------------------
+
+.. versionadded:: 2016.3.6,2016.11.3,Nitrogen
+
+Default: ``['libvirt', 'virtkey']``
+
+The external pillars permitted to be used on-demand using :py:func:`pillar.ext
+<salt.modules.pillar.ext>`.
+
+.. code-block:: yaml
+
+    on_demand_ext_pillar:
+      - libvirt
+      - virtkey
+      - git
+
+.. warning::
+    This will allow minions to request specific pillar data via
+    :py:func:`pillar.ext <salt.modules.pillar.ext>`, and may be considered a
+    security risk. However, pillar data generated in this way will not affect
+    the :ref:`in-memory pillar data <pillar-in-memory>`, so this risk is
+    limited to instances in which states/modules/etc. (built-in or custom) rely
+    upon pillar data generated by :py:func:`pillar.ext
+    <salt.modules.pillar.ext>`.
+
 .. conf_master:: pillar_opts
 
 ``pillar_opts``

doc/ref/configuration/minion.rst

@@ -1608,6 +1608,35 @@ the pillar environments.
       prod:
         - /srv/pillar/prod
 
+.. conf_minion:: on_demand_ext_pillar
+
+``on_demand_ext_pillar``
+------------------------
+
+.. versionadded:: 2016.3.6,2016.11.3,Nitrogen
+
+Default: ``['libvirt', 'virtkey']``
+
+When using a local :conf_minion:`file_client`, this option controls which
+external pillars are permitted to be used on-demand using :py:func:`pillar.ext
+<salt.modules.pillar.ext>`.
+
+.. code-block:: yaml
+
+    on_demand_ext_pillar:
+      - libvirt
+      - virtkey
+      - git
+
+.. warning::
+    This will allow a masterless minion to request specific pillar data via
+    :py:func:`pillar.ext <salt.modules.pillar.ext>`, and may be considered a
+    security risk. However, pillar data generated in this way will not affect
+    the :ref:`in-memory pillar data <pillar-in-memory>`, so this risk is
+    limited to instances in which states/modules/etc. (built-in or custom) rely
+    upon pillar data generated by :py:func:`pillar.ext
+    <salt.modules.pillar.ext>`.
+
 .. conf_minion:: pillarenv
 
 ``pillarenv``

pkg/osx/req.txt

@@ -11,7 +11,7 @@ GitPython==1.0.1
 idna==2.0
 ioflo==1.5.0
 ipaddress==1.0.16
-Jinja2==2.8
+Jinja2==2.9.4
 libnacl==1.4.4
 linode-python==1.1.1
 Mako==1.0.3

salt/auth/pam.py

@@ -23,6 +23,14 @@ authenticated against.  This defaults to `login`
 
     The Python interface to PAM does not support authenticating as ``root``.
 
+.. note:: Using PAM groups with SSSD groups on python2.
+
+    To use sssd with the PAM eauth module and groups the `pysss` module is
+    needed.  On RedHat/CentOS this is `python-sss`.
+
+    This should not be needed with python >= 3.3, because the `os` modules has the
+    `getgrouplist` function.
+
 '''
 
 # Import Python Libs

salt/config/__init__.py

@@ -254,6 +254,9 @@ VALID_OPTS = {
     # A map of saltenvs and fileserver backend locations
     'pillar_roots': dict,
 
+    # The external pillars permitted to be used on-demand using pillar.ext
+    'on_demand_ext_pillar': list,
+
     # The type of hashing algorithm to use when doing file comparisons
     'hash_type': str,
 
@@ -1004,6 +1007,7 @@ DEFAULT_MINION_OPTS = {
         'base': [salt.syspaths.BASE_PILLAR_ROOTS_DIR,
                  salt.syspaths.SPM_PILLAR_PATH]
     },
+    'on_demand_ext_pillar': ['libvirt', 'virtkey'],
     'git_pillar_base': 'master',
     'git_pillar_branch': 'master',
     'git_pillar_env': '',
@@ -1198,6 +1202,7 @@ DEFAULT_MASTER_OPTS = {
         'base': [salt.syspaths.BASE_PILLAR_ROOTS_DIR,
                  salt.syspaths.SPM_PILLAR_PATH]
     },
+    'on_demand_ext_pillar': ['libvirt', 'virtkey'],
     'thorium_interval': 0.5,
     'thorium_roots': {
         'base': [salt.syspaths.BASE_THORIUM_ROOTS_DIR],

salt/engines/logstash.py

@@ -19,7 +19,6 @@ them onto a logstash endpoint.
 # Import python libraries
 from __future__ import absolute_import
 import logging
-import json
 
 # Import salt libs
 import salt.utils.event
@@ -66,4 +65,4 @@ def start(host, port=5959, tag='salt/engine/logstash'):
     while True:
         event = event_bus.get_event()
         if event:
-            logstash_logger.info(tag, extra=json.dumps(event))
+            logstash_logger.info(tag, extra=event)

salt/master.py

@@ -1881,9 +1881,9 @@ class ClearFuncs(object):
                     'user': username}
 
             self.event.fire_event(data, tagify([jid, 'new'], 'wheel'))
-            ret = self.wheel_.call_func(fun, **clear_load)
-            data['return'] = ret
-            data['success'] = True
+            ret = self.wheel_.call_func(fun, full_return=True, **clear_load)
+            data['return'] = ret['return']
+            data['success'] = ret['success']
             self.event.fire_event(data, tagify([jid, 'ret'], 'wheel'))
             return {'tag': tag,
                     'data': data}

salt/modules/launchctl.py

@@ -103,11 +103,17 @@ def _available_services():
                         plist = plistlib.readPlistFromBytes(
                             salt.utils.to_bytes(plist_xml))
 
-                available_services[plist.Label.lower()] = {
-                    'filename': filename,
-                    'file_path': true_path,
-                    'plist': plist,
-                }
+                try:
+                    available_services[plist.Label.lower()] = {
+                        'filename': filename,
+                        'file_path': true_path,
+                        'plist': plist,
+                    }
+                except AttributeError:
+                    # As of MacOS 10.12 there might be plist files without Label key
+                    # in the searched directories. As these files do not represent
+                    # services, thay are not added to the list.
+                    pass
 
     return available_services
 

salt/modules/pillar.py

@@ -304,9 +304,39 @@ def raw(key=None):
 
 def ext(external, pillar=None):
     '''
+    .. versionchanged:: 2016.3.6,2016.11.3,Nitrogen
+        The supported ext_pillar types are now tunable using the
+        :conf_master:`on_demand_ext_pillar` config option. Earlier releases
+        used a hard-coded default.
+
     Generate the pillar and apply an explicit external pillar
 
-    CLI Example:
+
+    external
+        A single ext_pillar to add to the ext_pillar configuration. This must
+        be passed as a single section from the ext_pillar configuration (see
+        CLI examples below). For more complicated ``ext_pillar``
+        configurations, it can be helpful to use the Python shell to load YAML
+        configuration into a dictionary, and figure out
+
+        .. code-block:: python
+
+            >>> import yaml
+            >>> ext_pillar = yaml.safe_load("""
+            ... ext_pillar:
+            ...   - git:
+            ...     - issue38440 https://github.com/terminalmage/git_pillar:
+            ...       - env: base
+            ... """)
+            >>> ext_pillar
+            {'ext_pillar': [{'git': [{'mybranch https://github.com/myuser/myrepo': [{'env': 'base'}]}]}]}
+            >>> ext_pillar['ext_pillar'][0]
+            {'git': [{'mybranch https://github.com/myuser/myrepo': [{'env': 'base'}]}]}
+
+        In the above example, the value to pass would be
+        ``{'git': [{'mybranch https://github.com/myuser/myrepo': [{'env': 'base'}]}]}``.
+        Note that this would need to be quoted when passing on the CLI (as in
+        the CLI examples below).
 
     pillar : None
         If specified, allows for a dictionary of pillar data to be made
@@ -316,9 +346,13 @@ def ext(external, pillar=None):
 
         .. versionadded:: 2015.5.0
 
+    CLI Examples:
+
     .. code-block:: bash
 
         salt '*' pillar.ext '{libvirt: _}'
+        salt '*' pillar.ext "{'git': ['master https://github.com/myuser/myrepo']}"
+        salt '*' pillar.ext "{'git': [{'mybranch https://github.com/myuser/myrepo': [{'env': 'base'}]}]}"
     '''
     if isinstance(external, six.string_types):
         external = yaml.safe_load(external)

salt/pillar/__init__.py

@@ -261,10 +261,11 @@ class Pillar(object):
     def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None,
                  pillar=None, pillarenv=None):
         self.minion_id = minion_id
+        self.ext = ext
         # Store the file_roots path so we can restore later. Issue 5449
         self.actual_file_roots = opts['file_roots']
         # use the local file client
-        self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, ext=ext, pillarenv=pillarenv)
+        self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, pillarenv=pillarenv)
         self.saltenv = saltenv
         self.client = salt.fileclient.get_file_client(self.opts, True)
 
@@ -303,16 +304,39 @@ class Pillar(object):
             else:
                 log.error('Pillar data must be a dictionary')
 
-    def __valid_ext(self, ext):
+    def __valid_on_demand_ext_pillar(self, opts):
         '''
         Check to see if the on demand external pillar is allowed
         '''
-        if not isinstance(ext, dict):
-            return {}
-        valid = set(('libvirt', 'virtkey'))
-        if any(key not in valid for key in ext):
-            return {}
-        return ext
+        if not isinstance(self.ext, dict):
+            log.error(
+                'On-demand pillar %s is not formatted as a dictionary',
+                self.ext
+            )
+            return False
+
+        on_demand = opts.get('on_demand_ext_pillar', [])
+        try:
+            invalid_on_demand = set([x for x in self.ext if x not in on_demand])
+        except TypeError:
+            # Prevent traceback when on_demand_ext_pillar option is malformed
+            log.error(
+                'The \'on_demand_ext_pillar\' configuration option is '
+                'malformed, it should be a list of ext_pillar module names'
+            )
+            return False
+
+        if invalid_on_demand:
+            log.error(
+                'The following ext_pillar modules are not allowed for '
+                'on-demand pillar data: %s. Valid on-demand ext_pillar '
+                'modules are: %s. The valid modules can be adjusted by '
+                'setting the \'on_demand_ext_pillar\' config option.',
+                ', '.join(sorted(invalid_on_demand)),
+                ', '.join(on_demand),
+            )
+            return False
+        return True
 
     def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None):
         '''
@@ -336,11 +360,11 @@ class Pillar(object):
             opts['state_top'] = salt.utils.url.create(opts['state_top'][1:])
         else:
             opts['state_top'] = salt.utils.url.create(opts['state_top'])
-        if self.__valid_ext(ext):
+        if self.ext and self.__valid_on_demand_ext_pillar(opts):
             if 'ext_pillar' in opts:
-                opts['ext_pillar'].append(ext)
+                opts['ext_pillar'].append(self.ext)
             else:
-                opts['ext_pillar'] = [ext]
+                opts['ext_pillar'] = [self.ext]
         return opts
 
     def _get_envs(self):
@@ -727,6 +751,21 @@ class Pillar(object):
         '''
         if errors is None:
             errors = []
+        try:
+            # Make sure that on-demand git_pillar is fetched before we try to
+            # compile the pillar data. git_pillar will fetch a remote when
+            # the git ext_pillar() func is run, but only for masterless.
+            if self.ext and 'git' in self.ext \
+                    and self.opts.get('__role') != 'minion':
+                # Avoid circular import
+                import salt.utils.gitfs
+                from salt.pillar.git_pillar import PER_REMOTE_OVERRIDES
+                git_pillar = salt.utils.gitfs.GitPillar(self.opts)
+                git_pillar.init_remotes(self.ext['git'], PER_REMOTE_OVERRIDES)
+                git_pillar.fetch_remotes()
+        except TypeError:
+            # Handle malformed ext_pillar
+            pass
         if 'ext_pillar' not in self.opts:
             return pillar, errors
         if not isinstance(self.opts['ext_pillar'], list):

salt/utils/__init__.py

@@ -1077,10 +1077,10 @@ def format_call(fun,
             continue
         extra[key] = copy.deepcopy(value)
 
-    # We'll be showing errors to the users until Salt Nitrogen comes out, after
+    # We'll be showing errors to the users until Salt Oxygen comes out, after
     # which, errors will be raised instead.
     warn_until(
-        'Nitrogen',
+        'Oxygen',
         'It\'s time to start raising `SaltInvocationError` instead of '
         'returning warnings',
         # Let's not show the deprecation warning on the console, there's no
@@ -1117,7 +1117,7 @@ def format_call(fun,
             '{0}. If you were trying to pass additional data to be used '
             'in a template context, please populate \'context\' with '
             '\'key: value\' pairs. Your approach will work until Salt '
-            'Nitrogen is out.{1}'.format(
+            'Oxygen is out.{1}'.format(
                 msg,
                 '' if 'full' not in ret else ' Please update your state files.'
             )

salt/utils/process.py

@@ -696,8 +696,17 @@ class SignalHandlingMultiprocessingProcess(MultiprocessingProcess):
 def default_signals(*signals):
     old_signals = {}
     for signum in signals:
-        old_signals[signum] = signal.getsignal(signum)
-        signal.signal(signum, signal.SIG_DFL)
+        try:
+            signal.signal(signum, signal.SIG_DFL)
+            old_signals[signum] = signal.getsignal(signum)
+        except ValueError as exc:
+            # This happens when a netapi module attempts to run a function
+            # using wheel_async, because the process trying to register signals
+            # will not be the main PID.
+            log.trace(
+                'Failed to register signal for signum %d: %s',
+                signum, exc
+            )
 
     # Do whatever is needed with the reset signals
     yield

tests/unit/modules/boto_vpc_test.py

@@ -517,8 +517,9 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin):
         '''
         Tests describing parameters via vpc id if vpc exist
         '''
-        # With moto 0.4.25 is_default is set to True. 0.4.24 and older, is_default is False
-        if _get_moto_version() >= LooseVersion('0.4.25'):
+        # With moto 0.4.25 through 0.4.30, is_default is set to True.
+        # 0.4.24 and older and 0.4.31 and newer, is_default is False
+        if LooseVersion('0.4.25') <= _get_moto_version() < LooseVersion('0.4.31'):
             is_default = True
         else:
             is_default = False