Merge pull request #48393 from FraaJad/develop

consul_pillar.py disable yaml loader and nest in pillar subkey

salt/pillar/consul_pillar.py

@@ -119,6 +119,21 @@ Matchers for more examples.
     ext_pillar:
       - consul: my_consul_config root=salt target="L@salt.example.com and G@osarch:x86_64"
 
+The data from Consul can be merged into a nested key in Pillar.
+
+.. code-block:: yaml
+
+    ext_pillar:
+      - consul: my_consul_config pillar_root=consul_data
+
+By default, keys containing YAML data will be deserialized before being merged into Pillar.
+This behavior can be disabled by setting ``expand_keys`` to ``false``.
+
+.. code-block:: yaml
+
+    ext_pillar:
+      - consul: my_consul_config expand_keys=false
+
 '''
 from __future__ import absolute_import, print_function, unicode_literals
 
@@ -168,6 +183,7 @@ def ext_pillar(minion_id,
         checker = salt.utils.minions.CkMinions(__opts__)
         _res = checker.check_minions(opts['target'], 'compound')
         minions = _res['minions']
+        log.debug('Targeted minions: %r', minions)
         if minion_id not in minions:
             return {}
 
@@ -179,6 +195,14 @@ def ext_pillar(minion_id,
     else:
         opts['root'] = ""
 
+    pillar_root_re = re.compile('pillar_root=(\S*)')  # pylint: disable=W1401
+    match = pillar_root_re.search(temp)
+    if match:
+        opts['pillar_root'] = match.group(1)
+        temp = temp.replace(match.group(0), '')
+    else:
+        opts['pillar_root'] = ""
+
     profile_re = re.compile('(?:profile=)?(\S+)')  # pylint: disable=W1401
     match = profile_re.search(temp)
     if match:
@@ -187,6 +211,14 @@ def ext_pillar(minion_id,
     else:
         opts['profile'] = None
 
+    expand_keys_re = re.compile('expand_keys=False', re.IGNORECASE)  # pylint: disable=W1401
+    match = expand_keys_re.search(temp)
+    if match:
+        opts['expand_keys'] = False
+        temp = temp.replace(match.group(0), '')
+    else:
+        opts['expand_keys'] = True
+
     client = get_conn(__opts__, opts['profile'])
 
     role = __salt__['grains.get']('role', None)
@@ -199,7 +231,22 @@ def ext_pillar(minion_id,
     }
 
     try:
-        pillar = fetch_tree(client, opts['root'])
+        pillar_tree = fetch_tree(client, opts['root'], opts['expand_keys'])
+        if opts['pillar_root']:
+            log.debug('Merging consul path %s/ into pillar at %s/', opts['root'], opts['pillar_root'])
+
+            pillar = {}
+            branch = pillar
+            keys = opts['pillar_root'].rstrip('/').split('/')
+
+            for i, k in enumerate(keys):
+                if i == len(keys) - 1:
+                    branch[k] = pillar_tree
+                else:
+                    branch[k] = {}
+                    branch = branch[k]
+        else:
+            pillar = pillar_tree
     except KeyError:
         log.error('No such key in consul profile %s: %s', opts['profile'], opts['root'])
         pillar = {}
@@ -214,13 +261,13 @@ def consul_fetch(client, path):
     return client.kv.get(path, recurse=True)
 
 
-def fetch_tree(client, path):
+def fetch_tree(client, path, expand_keys):
     '''
     Grab data from consul, trim base path and remove any keys which
     are folders. Take the remaining data and send it to be formatted
     in such a way as to be used as pillar data.
     '''
-    index, items = consul_fetch(client, path)
+    _, items = consul_fetch(client, path)
     ret = {}
     has_children = re.compile(r'/$')
 
@@ -234,13 +281,13 @@ def fetch_tree(client, path):
             log.debug('key/path - %s: %s', path, key)
             log.debug('has_children? %r', has_children.search(key))
         if has_children.search(key) is None:
-            ret = pillar_format(ret, key.split('/'), item['Value'])
+            ret = pillar_format(ret, key.split('/'), item['Value'], expand_keys)
             log.debug('Fetching subkeys for key: %r', item)
 
     return ret
 
 
-def pillar_format(ret, keys, value):
+def pillar_format(ret, keys, value, expand_keys):
     '''
     Perform data formatting to be used as pillar data and
     merge it with the current pillar data
@@ -250,9 +297,12 @@ def pillar_format(ret, keys, value):
         return ret
 
     # If value is not None then it's a string
-    # Use YAML to parse the data
     # YAML strips whitespaces unless they're surrounded by quotes
-    pillar_value = salt.utils.yaml.safe_load(value)
+    # If expand_keys is true, deserialize the YAML data
+    if expand_keys:
+        pillar_value = salt.utils.yaml.safe_load(value)
+    else:
+        pillar_value = value
 
     keyvalue = keys.pop()
     pil = {keyvalue: pillar_value}

tests/unit/pillar/test_consul.py

@@ -29,6 +29,7 @@ PILLAR_DATA = [
     {'Value': 'Test User', 'Key': 'test-shared/user/full_name'},
     {'Value': 'adm\nwww-data\nmlocate', 'Key': 'test-shared/user/groups'},
     {'Value': '"adm\nwww-data\nmlocate"', 'Key': 'test-shared/user/dontsplit'},
+    {'Value': 'yaml:\n  key: value\n', 'Key': 'test-shared/user/dontexpand'},
     {'Value': None, 'Key': 'test-shared/user/blankvalue'},
     {'Value': 'test', 'Key': 'test-shared/user/login'},
     {'Value': None, 'Key': 'test-shared/user/'}
@@ -65,12 +66,30 @@ class ConsulPillarTestCase(TestCase, LoaderModuleMockMixin):
                 assert sorted(pillar_data) == ['sites', 'user']
                 self.assertNotIn('blankvalue', pillar_data['user'])
 
+    def test_pillar_nest(self):
+        with patch.dict(consul_pillar.__salt__, {'grains.get': MagicMock(return_value=({}))}):
+            with patch.object(consul_pillar, 'consul_fetch', MagicMock(return_value=('2232', PILLAR_DATA))):
+                pillar_data = consul_pillar.ext_pillar(
+                    'testminion', {}, 'consul_config root=test-shared/ pillar_root=nested-key/'
+                )
+                consul_pillar.consul_fetch.assert_called_once_with('consul_connection', 'test-shared/')
+                assert sorted(pillar_data['nested-key']) == ['sites', 'user']
+                self.assertNotIn('blankvalue', pillar_data['nested-key']['user'])
+
     def test_value_parsing(self):
         with patch.dict(consul_pillar.__salt__, {'grains.get': MagicMock(return_value=({}))}):
             with patch.object(consul_pillar, 'consul_fetch', MagicMock(return_value=('2232', PILLAR_DATA))):
                 pillar_data = consul_pillar.ext_pillar('testminion', {}, 'consul_config root=test-shared/')
                 assert isinstance(pillar_data['user']['dontsplit'], six.string_types)
 
+    def test_non_expansion(self):
+        with patch.dict(consul_pillar.__salt__, {'grains.get': MagicMock(return_value=({}))}):
+            with patch.object(consul_pillar, 'consul_fetch', MagicMock(return_value=('2232', PILLAR_DATA))):
+                pillar_data = consul_pillar.ext_pillar(
+                    'testminion', {}, 'consul_config root=test-shared/ expand_keys=false'
+                )
+                assert isinstance(pillar_data['user']['dontexpand'], six.string_types)
+
     def test_dict_merge(self):
         test_dict = {}
         with patch.dict(test_dict, SIMPLE_DICT):