Merge pull request #38952 from terminalmage/zd1168

Make the ext_pillars available to pillar.ext tunable

conf/master

@@ -326,7 +326,7 @@
 # publisher_acl_blacklist instead.
 
 # Enforce publisher_acl & publisher_acl_blacklist when users have sudo
-# access to the salt command. 
+# access to the salt command.
 #
 #sudo_acl: False
 
@@ -520,7 +520,7 @@
 # WARNING: While md5 is supported, do not use it due to the high chance
 # of possible collisions and thus security breach.
 #
-# Prior to changing this value, the master should be stopped and all Salt 
+# Prior to changing this value, the master should be stopped and all Salt
 # caches should be cleared.
 #hash_type: md5
 
@@ -597,20 +597,20 @@
 # Along with gitfs_password, is used to authenticate to HTTPS remotes.
 # gitfs_user: ''
 
-# Along with gitfs_user, is used to authenticate to HTTPS remotes. 
+# Along with gitfs_user, is used to authenticate to HTTPS remotes.
 # This parameter is not required if the repository does not use authentication.
 #gitfs_password: ''
 
-# By default, Salt will not authenticate to an HTTP (non-HTTPS) remote. 
+# By default, Salt will not authenticate to an HTTP (non-HTTPS) remote.
 # This parameter enables authentication over HTTP. Enable this at your own risk.
 #gitfs_insecure_auth: False
 
-# Along with gitfs_privkey (and optionally gitfs_passphrase), is used to 
+# Along with gitfs_privkey (and optionally gitfs_passphrase), is used to
 # authenticate to SSH remotes. This parameter (or its per-remote counterpart)
 # is required for SSH remotes.
 #gitfs_pubkey: ''
 
-# Along with gitfs_pubkey (and optionally gitfs_passphrase), is used to 
+# Along with gitfs_pubkey (and optionally gitfs_passphrase), is used to
 # authenticate to SSH remotes. This parameter (or its per-remote counterpart)
 # is required for SSH remotes.
 #gitfs_privkey: ''
@@ -665,6 +665,11 @@
 # ext_pillar.
 #ext_pillar_first: False
 
+# The external pillars permitted to be used on-demand using pillar.ext
+#on_demand_ext_pillar:
+#  - libvirt
+#  - virtkey
+
 # The pillar_gitfs_ssl_verify option specifies whether to ignore ssl certificate
 # errors when contacting the pillar gitfs backend. You might want to set this to
 # false if you're using a git backend that uses a self-signed certificate but
@@ -713,7 +718,7 @@
 # be used instead
 #git_pillar_branch: master
 
-# Environment to use for git_pillar remotes. This is normally derived from 
+# Environment to use for git_pillar remotes. This is normally derived from
 # the branch/tag (or from a per-remote env parameter), but if set this will
 # override the process of deriving the env from the branch/tag name.
 #git_pillar_env: ''
@@ -723,12 +728,12 @@
 #git_pillar_root: ''
 
 # Specifies whether or not to ignore SSL certificate errors when contacting
-# the remote repository. 
+# the remote repository.
 #git_pillar_ssl_verify: False
 
 # When set to False, if there is an update/checkout lock for a git_pillar
 # remote and the pid written to it is not running on the master, the lock
-# file will be automatically cleared and a new lock will be obtained. 
+# file will be automatically cleared and a new lock will be obtained.
 #git_pillar_global_lock: True
 
 # Git External Pillar Authentication Options

conf/minion

@@ -327,7 +327,7 @@
 
 # Grains cache expiration, in seconds. If the cache file is older than this
 # number of seconds then the grains cache will be dumped and fully re-populated
-# with fresh data. Defaults to 5 minutes. Will have no effect if 'grains_cache' 
+# with fresh data. Defaults to 5 minutes. Will have no effect if 'grains_cache'
 # is not enabled.
 # grains_cache_expiration: 300
 

doc/ref/configuration/master.rst

@@ -2282,6 +2282,34 @@ configuration is the same as :conf_master:`file_roots`:
       prod:
         - /srv/pillar/prod
 
+.. conf_master:: on_demand_ext_pillar
+
+``on_demand_ext_pillar``
+------------------------
+
+.. versionadded:: 2016.3.6,2016.11.3,Nitrogen
+
+Default: ``['libvirt', 'virtkey']``
+
+The external pillars permitted to be used on-demand using :py:func:`pillar.ext
+<salt.modules.pillar.ext>`.
+
+.. code-block:: yaml
+
+    on_demand_ext_pillar:
+      - libvirt
+      - virtkey
+      - git
+
+.. warning::
+    This will allow minions to request specific pillar data via
+    :py:func:`pillar.ext <salt.modules.pillar.ext>`, and may be considered a
+    security risk. However, pillar data generated in this way will not affect
+    the :ref:`in-memory pillar data <pillar-in-memory>`, so this risk is
+    limited to instances in which states/modules/etc. (built-in or custom) rely
+    upon pillar data generated by :py:func:`pillar.ext
+    <salt.modules.pillar.ext>`.
+
 .. conf_master:: pillar_opts
 
 ``pillar_opts``

doc/ref/configuration/minion.rst

@@ -1454,6 +1454,35 @@ the pillar environments.
       prod:
         - /srv/pillar/prod
 
+.. conf_minion:: on_demand_ext_pillar
+
+``on_demand_ext_pillar``
+------------------------
+
+.. versionadded:: 2016.3.6,2016.11.3,Nitrogen
+
+Default: ``['libvirt', 'virtkey']``
+
+When using a local :conf_minion:`file_client`, this option controls which
+external pillars are permitted to be used on-demand using :py:func:`pillar.ext
+<salt.modules.pillar.ext>`.
+
+.. code-block:: yaml
+
+    on_demand_ext_pillar:
+      - libvirt
+      - virtkey
+      - git
+
+.. warning::
+    This will allow a masterless minion to request specific pillar data via
+    :py:func:`pillar.ext <salt.modules.pillar.ext>`, and may be considered a
+    security risk. However, pillar data generated in this way will not affect
+    the :ref:`in-memory pillar data <pillar-in-memory>`, so this risk is
+    limited to instances in which states/modules/etc. (built-in or custom) rely
+    upon pillar data generated by :py:func:`pillar.ext
+    <salt.modules.pillar.ext>`.
+
 .. conf_minion:: pillarenv
 
 ``pillarenv``

salt/config/__init__.py

@@ -237,6 +237,9 @@ VALID_OPTS = {
     # A map of saltenvs and fileserver backend locations
     'pillar_roots': dict,
 
+    # The external pillars permitted to be used on-demand using pillar.ext
+    'on_demand_ext_pillar': list,
+
     # The type of hashing algorithm to use when doing file comparisons
     'hash_type': str,
 
@@ -938,6 +941,7 @@ DEFAULT_MINION_OPTS = {
         'base': [salt.syspaths.BASE_PILLAR_ROOTS_DIR,
                  salt.syspaths.SPM_PILLAR_PATH]
     },
+    'on_demand_ext_pillar': ['libvirt', 'virtkey'],
     'git_pillar_base': 'master',
     'git_pillar_branch': 'master',
     'git_pillar_env': '',
@@ -1111,6 +1115,7 @@ DEFAULT_MASTER_OPTS = {
         'base': [salt.syspaths.BASE_PILLAR_ROOTS_DIR,
                  salt.syspaths.SPM_PILLAR_PATH]
     },
+    'on_demand_ext_pillar': ['libvirt', 'virtkey'],
     'thorium_interval': 0.5,
     'thorium_roots': {
         'base': [salt.syspaths.BASE_THORIUM_ROOTS_DIR],

salt/modules/pillar.py

@@ -304,9 +304,39 @@ def raw(key=None):
 
 def ext(external, pillar=None):
     '''
+    .. versionchanged:: 2016.3.6,2016.11.3,Nitrogen
+        The supported ext_pillar types are now tunable using the
+        :conf_master:`on_demand_ext_pillar` config option. Earlier releases
+        used a hard-coded default.
+
     Generate the pillar and apply an explicit external pillar
 
-    CLI Example:
+
+    external
+        A single ext_pillar to add to the ext_pillar configuration. This must
+        be passed as a single section from the ext_pillar configuration (see
+        CLI examples below). For more complicated ``ext_pillar``
+        configurations, it can be helpful to use the Python shell to load YAML
+        configuration into a dictionary, and figure out
+
+        .. code-block:: python
+
+            >>> import yaml
+            >>> ext_pillar = yaml.safe_load("""
+            ... ext_pillar:
+            ...   - git:
+            ...     - issue38440 https://github.com/terminalmage/git_pillar:
+            ...       - env: base
+            ... """)
+            >>> ext_pillar
+            {'ext_pillar': [{'git': [{'mybranch https://github.com/myuser/myrepo': [{'env': 'base'}]}]}]}
+            >>> ext_pillar['ext_pillar'][0]
+            {'git': [{'mybranch https://github.com/myuser/myrepo': [{'env': 'base'}]}]}
+
+        In the above example, the value to pass would be
+        ``{'git': [{'mybranch https://github.com/myuser/myrepo': [{'env': 'base'}]}]}``.
+        Note that this would need to be quoted when passing on the CLI (as in
+        the CLI examples below).
 
     pillar : None
         If specified, allows for a dictionary of pillar data to be made
@@ -316,9 +346,13 @@ def ext(external, pillar=None):
 
         .. versionadded:: 2015.5.0
 
+    CLI Examples:
+
     .. code-block:: bash
 
         salt '*' pillar.ext '{libvirt: _}'
+        salt '*' pillar.ext "{'git': ['master https://github.com/myuser/myrepo']}"
+        salt '*' pillar.ext "{'git': [{'mybranch https://github.com/myuser/myrepo': [{'env': 'base'}]}]}"
     '''
     if isinstance(external, six.string_types):
         external = yaml.safe_load(external)

salt/pillar/__init__.py

@@ -275,10 +275,11 @@ class Pillar(object):
     def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None,
                  pillar=None, pillarenv=None):
         self.minion_id = minion_id
+        self.ext = ext
         # Store the file_roots path so we can restore later. Issue 5449
         self.actual_file_roots = opts['file_roots']
         # use the local file client
-        self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, ext=ext, pillarenv=pillarenv)
+        self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, pillarenv=pillarenv)
         self.saltenv = saltenv
         self.client = salt.fileclient.get_file_client(self.opts, True)
 
@@ -317,18 +318,41 @@ class Pillar(object):
             else:
                 log.error('Pillar data must be a dictionary')
 
-    def __valid_ext(self, ext):
+    def __valid_on_demand_ext_pillar(self, opts):
         '''
         Check to see if the on demand external pillar is allowed
         '''
-        if not isinstance(ext, dict):
-            return {}
-        valid = set(('libvirt', 'virtkey'))
-        if any(key not in valid for key in ext):
-            return {}
-        return ext
+        if not isinstance(self.ext, dict):
+            log.error(
+                'On-demand pillar %s is not formatted as a dictionary',
+                self.ext
+            )
+            return False
 
-    def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, env=None, pillarenv=None):
+        on_demand = opts.get('on_demand_ext_pillar', [])
+        try:
+            invalid_on_demand = set([x for x in self.ext if x not in on_demand])
+        except TypeError:
+            # Prevent traceback when on_demand_ext_pillar option is malformed
+            log.error(
+                'The \'on_demand_ext_pillar\' configuration option is '
+                'malformed, it should be a list of ext_pillar module names'
+            )
+            return False
+
+        if invalid_on_demand:
+            log.error(
+                'The following ext_pillar modules are not allowed for '
+                'on-demand pillar data: %s. Valid on-demand ext_pillar '
+                'modules are: %s. The valid modules can be adjusted by '
+                'setting the \'on_demand_ext_pillar\' config option.',
+                ', '.join(sorted(invalid_on_demand)),
+                ', '.join(on_demand),
+            )
+            return False
+        return True
+
+    def __gen_opts(self, opts_in, grains, saltenv=None, env=None, pillarenv=None):
         '''
         The options need to be altered to conform to the file client
         '''
@@ -359,11 +383,11 @@ class Pillar(object):
             opts['state_top'] = salt.utils.url.create(opts['state_top'][1:])
         else:
             opts['state_top'] = salt.utils.url.create(opts['state_top'])
-        if self.__valid_ext(ext):
+        if self.ext and self.__valid_on_demand_ext_pillar(opts):
             if 'ext_pillar' in opts:
-                opts['ext_pillar'].append(ext)
+                opts['ext_pillar'].append(self.ext)
             else:
-                opts['ext_pillar'] = [ext]
+                opts['ext_pillar'] = [self.ext]
         return opts
 
     def _get_envs(self):
@@ -742,6 +766,21 @@ class Pillar(object):
         '''
         if errors is None:
             errors = []
+        try:
+            # Make sure that on-demand git_pillar is fetched before we try to
+            # compile the pillar data. git_pillar will fetch a remote when
+            # the git ext_pillar() func is run, but only for masterless.
+            if self.ext and 'git' in self.ext \
+                    and self.opts.get('__role') != 'minion':
+                # Avoid circular import
+                import salt.utils.gitfs
+                from salt.pillar.git_pillar import PER_REMOTE_OVERRIDES
+                git_pillar = salt.utils.gitfs.GitPillar(self.opts)
+                git_pillar.init_remotes(self.ext['git'], PER_REMOTE_OVERRIDES)
+                git_pillar.fetch_remotes()
+        except TypeError:
+            # Handle malformed ext_pillar
+            pass
         if 'ext_pillar' not in self.opts:
             return pillar, errors
         if not isinstance(self.opts['ext_pillar'], list):