saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #30458 from rallytime/bp-30062

Browse source 
Back-port #30062 to 2015.8
This commit is contained in:
Mike Place 2016-01-20 09:40:22 -07:00
commit 73f372dc98
10 changed files with 55 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
conf/master
View file

@ -614,6 +614,9 @@
# on the "renderer" setting and is the default value.
#pillar_source_merging_strategy: smart
# Recursively merge lists by aggregating them instead of replacing them.
#pillar_merge_lists: False
##### Syndic settings #####
##########################################

44
doc/ref/configuration/master.rst
View file

@ -2253,35 +2253,6 @@ strategy between different sources. It accepts 4 values:
element2: True
baz: quux
* recurse_list:
it will merge recursively mapping of data similar to ``recurse`` but merge
lists by aggregating them instead of replacing them.
.. code-block:: yaml
foo: 43
bar:
- 1
- 2
.. code-block:: yaml
bar:
- 3
baz: quux
will be merged as:
.. code-block:: yaml
foo: 42
bar:
- 1
- 2
- 3
baz: quux
* aggregate:
instructs aggregation of elements between sources that use the #!yamlex renderer.
@ -2351,6 +2322,21 @@ strategy between different sources. It accepts 4 values:
Guesses the best strategy based on the "renderer" setting.
``pillar_merge_lists``
----------------------------------
.. versionadded:: 2015.8.0
Default: ``True``
Recursively merge lists by aggregating them instead of replacing them.
.. code-block:: yaml
pillar_merge_lists: False
.. conf_master:: pillar_source_merging_strategy
Syndic Server Settings
======================

4
salt/auth/__init__.py
View file

@ -211,12 +211,14 @@ class Authorize(object):
Gather and create the authorization data sets
'''
auth_data = self.opts['external_auth']
merge_lists = self.opts['pillar_merge_lists']
if 'django' in auth_data and '^model' in auth_data['django']:
auth_from_django = salt.auth.django.retrieve_auth_entries()
auth_data = salt.utils.dictupdate.merge(auth_data,
auth_from_django,
strategy='list')
strategy='list',
merge_lists=merge_lists)
#for auth_back in self.opts.get('external_auth_sources', []):
# fstr = '{0}.perms'.format(auth_back)

4
salt/config.py
View file

@ -505,6 +505,9 @@ VALID_OPTS = {
# encountering duplicate values
'pillar_source_merging_strategy': str,
# Recursively merge lists by aggregating them instead of replacing them.
'pillar_merge_lists': bool,
# How to merge multiple top files from multiple salt environments
# (saltenvs); can be 'merge' or 'same'
'top_file_merging_strategy': str,
@ -1056,6 +1059,7 @@ DEFAULT_MASTER_OPTS = {
'pillar_opts': False,
'pillar_safe_render_error': True,
'pillar_source_merging_strategy': 'smart',
'pillar_merge_lists': False,
'ping_on_rotate': False,
'peer': {},
'preserve_minion_cache': False,

9
salt/modules/config.py
View file

@ -11,6 +11,7 @@ import os
import logging
# Import salt libs
import salt.config
import salt.utils
try:
# Gated for salt-ssh (salt.utils.cloud imports msgpack)
@ -357,10 +358,12 @@ def get(key, default='', delimiter=':', merge=None):
'to \'recurse\'.'.format(merge))
merge = 'recurse'
merge_lists = salt.config.master_config('/etc/salt/master').get('pillar_merge_lists')
data = copy.copy(__pillar__.get('master', {}))
data = salt.utils.dictupdate.merge(data, __pillar__, strategy=merge)
data = salt.utils.dictupdate.merge(data, __grains__, strategy=merge)
data = salt.utils.dictupdate.merge(data, __opts__, strategy=merge)
data = salt.utils.dictupdate.merge(data, __pillar__, strategy=merge, merge_lists=merge_lists)
data = salt.utils.dictupdate.merge(data, __grains__, strategy=merge, merge_lists=merge_lists)
data = salt.utils.dictupdate.merge(data, __opts__, strategy=merge, merge_lists=merge_lists)
ret = salt.utils.traverse_dict_and_list(data,
key,
'_|-',

12
salt/pillar/__init__.py
View file

@ -532,7 +532,8 @@ class Pillar(object):
state,
nstate,
self.merge_strategy,
self.opts.get('renderer', 'yaml'))
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', 'False'))
if err:
errors += err
@ -569,7 +570,8 @@ class Pillar(object):
pillar,
pstate,
self.merge_strategy,
self.opts.get('renderer', 'yaml'))
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', 'False'))
return pillar, errors
@ -655,7 +657,8 @@ class Pillar(object):
pillar,
ext,
self.merge_strategy,
self.opts.get('renderer', 'yaml'))
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', 'False'))
ext = None
return pillar
@ -672,7 +675,8 @@ class Pillar(object):
pillar = merge(pillar,
self.opts['pillar'],
self.merge_strategy,
self.opts.get('renderer', 'yaml'))
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', 'False'))
else:
matches = self.top_matches(top)
pillar, errors = self.render_pillar(matches)

7
salt/pillar/git_pillar.py
View file

@ -260,6 +260,10 @@ def ext_pillar(minion_id, repo, pillar_dirs):
'pillar_source_merging_strategy',
'smart'
)
merge_lists = __opts__.get(
'pillar_merge_lists',
'False'
)
for pillar_dir, env in six.iteritems(pillar.pillar_dirs):
log.debug(
'git_pillar is processing pillar SLS from {0} for pillar '
@ -273,7 +277,8 @@ def ext_pillar(minion_id, repo, pillar_dirs):
ret = salt.utils.dictupdate.merge(
ret,
local_pillar.compile_pillar(ext=False),
strategy=merge_strategy
strategy=merge_strategy,
merge_lists=merge_lists
)
return ret

12
salt/utils/dictupdate.py
View file

@ -85,14 +85,14 @@ def merge_aggregate(obj_a, obj_b):
return _yamlex_merge_recursive(obj_a, obj_b, level=1)
def merge_overwrite(obj_a, obj_b):
def merge_overwrite(obj_a, obj_b, merge_lists=False):
for obj in obj_b:
if obj in obj_a:
obj_a[obj] = obj_b[obj]
return merge_recurse(obj_a, obj_b)
return merge_recurse(obj_a, obj_b, merge_lists=merge_lists)
def merge(obj_a, obj_b, strategy='smart', renderer='yaml'):
def merge(obj_a, obj_b, strategy='smart', renderer='yaml', merge_lists=False):
if strategy == 'smart':
if renderer == 'yamlex' or renderer.startswith('yamlex_'):
strategy = 'aggregate'
@ -102,14 +102,12 @@ def merge(obj_a, obj_b, strategy='smart', renderer='yaml'):
if strategy == 'list':
merged = merge_list(obj_a, obj_b)
elif strategy == 'recurse':
merged = merge_recurse(obj_a, obj_b)
elif strategy == 'recurse_list':
merged = merge_recurse(obj_a, obj_b, merge_lists=True)
merged = merge_recurse(obj_a, obj_b, merge_lists)
elif strategy == 'aggregate':
#: level = 1 merge at least root data
merged = merge_aggregate(obj_a, obj_b)
elif strategy == 'overwrite':
merged = merge_overwrite(obj_a, obj_b)
merged = merge_overwrite(obj_a, obj_b, merge_lists)
else:
log.warning('Unknown merging strategy \'{0}\', '
'fallback to recurse'.format(strategy))

59
tests/unit/pillar_test.py
View file

@ -130,38 +130,6 @@ class PillarTestCase(TestCase):
pillar = salt.pillar.Pillar(opts, grains, 'mocked-minion', 'base')
self.assertEqual(pillar.compile_pillar()['ssh'], 'foo')
@patch('salt.pillar.salt.fileclient.get_file_client', autospec=True)
@patch('salt.pillar.salt.minion.Matcher') # autospec=True disabled due to py3 mock bug
def test_pillar_multiple_matches(self, Matcher, get_file_client):
# Uses the ``recurse_list`` strategy.
opts = {
'renderer': 'yaml',
'state_top': '',
'pillar_roots': [],
'extension_modules': '',
'environment': 'base',
'file_roots': [],
'pillar_source_merging_strategy': 'recurse_list',
}
grains = {
'os': 'Ubuntu',
'os_family': 'Debian',
'oscodename': 'raring',
'osfullname': 'Ubuntu',
'osrelease': '13.04',
'kernel': 'Linux'
}
self._setup_test_topfile_mocks(Matcher, get_file_client, 1, 2)
pillar = salt.pillar.Pillar(opts, grains, 'mocked-minion', 'base')
# Pillars should be merged, but only once per pillar file.
self.assertDictEqual(pillar.compile_pillar()['generic'], {
'key1': ['value1', 'value2', 'value3'],
'key2': {
'sub_key1': [],
'sub_key2': [],
}
})
def _setup_test_topfile_mocks(self, Matcher, get_file_client,
nodegroup_order, glob_order):
# Write a simple topfile and two pillar state files
@ -232,33 +200,6 @@ generic:
client.get_state.side_effect = get_state
@patch('salt.pillar.salt.fileclient.get_file_client', autospec=True)
@patch('salt.pillar.salt.minion.Matcher') # autospec=True disabled due to py3 mock bug
def test_pillar_include(self, Matcher, get_file_client):
# Uses the ``recurse_list`` strategy.
opts = {
'renderer': 'yaml',
'state_top': '',
'pillar_roots': [],
'extension_modules': '',
'environment': 'base',
'file_roots': [],
'pillar_source_merging_strategy': 'recurse_list',
}
grains = {
'os': 'Ubuntu',
'os_family': 'Debian',
'oscodename': 'raring',
'osfullname': 'Ubuntu',
'osrelease': '13.04',
'kernel': 'Linux'
}
self._setup_test_include_mocks(Matcher, get_file_client)
pillar = salt.pillar.Pillar(opts, grains, 'mocked-minion', 'base').compile_pillar()
# Both pillar modules should only be loaded once.
self.assertEqual(pillar['p1'], ['value1_3', 'value1_1', 'value1_2'])
self.assertEqual(pillar['p2'], ['value2_1', 'value2_2'])
def _setup_test_include_mocks(self, Matcher, get_file_client):
self.top_file = top_file = tempfile.NamedTemporaryFile()
top_file.write(b'''

8
tests/unit/utils/dictupdate_test.py
View file

@ -29,7 +29,8 @@ class UtilDictupdateTestCase(TestCase):
# level 1 value changes (list replacement)
mdict = copy.deepcopy(self.dict1)
mdict['A'] = [1, 2]
res = dictupdate.update(copy.deepcopy(mdict), {'A': [2, 3]})
res = dictupdate.update(copy.deepcopy(mdict), {'A': [2, 3]},
merge_lists=False)
mdict['A'] = [2, 3]
self.assertEqual(res, mdict)
@ -50,7 +51,8 @@ class UtilDictupdateTestCase(TestCase):
# level 2 value changes (list replacement)
mdict = copy.deepcopy(self.dict1)
mdict['C']['D'] = ['a', 'b']
res = dictupdate.update(copy.deepcopy(mdict), {'C': {'D': ['c', 'd']}})
res = dictupdate.update(copy.deepcopy(mdict), {'C': {'D': ['c', 'd']}},
merge_lists=False)
mdict['C']['D'] = ['c', 'd']
self.assertEqual(res, mdict)
@ -75,7 +77,7 @@ class UtilDictupdateTestCase(TestCase):
mdict = copy.deepcopy(self.dict1)
mdict['C']['F']['G'] = ['a', 'b']
res = dictupdate.update(copy.deepcopy(mdict),
{'C': {'F': {'G': ['c', 'd']}}})
{'C': {'F': {'G': ['c', 'd']}}}, merge_lists=False)
mdict['C']['F']['G'] = ['c', 'd']
self.assertEqual(res, mdict)