saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Improve filter_by() base argument:

Browse source 
* Change filter_by() base argument to a key in lookup_dict rather
    than a dictionary.  This allows more compact usage and more
    readable use of base values/defaults in the lookup_dict.

  * Change/improve function-local documentation for lookup_dict().

  * Add specific use-case of "base" argument for lookup_dict() in the
    formulas documentation.

  * Expand filter_by() unit testing to validate use of base argument.

  * Add unit test for salt.utils.dictupdate.update() to provide
    complete depth-of-testing of filter_by() which uses
    dictupdate.update().
This commit is contained in:
Thayne Harbaugh 2014-10-24 11:13:47 -06:00
parent 50d51e76e0
commit 65e477dd67
5 changed files with 220 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
doc/topics/development/conventions/formulas.rst
View file

@ -259,7 +259,7 @@ syntax for referencing a value is a normal dictionary lookup in Jinja, such as
},
'Gentoo': {
'server': 'dev-db/mysql',
'mysql-client': 'dev-db/mysql',
'client': 'dev-db/mysql',
'service': 'mysql',
'config': '/etc/mysql/my.cnf',
'python': 'dev-python/mysql-python',
@ -281,6 +281,44 @@ state file using the following syntax:
- running
- name: {{ mysql.service }}
Collecting common values
````````````````````````
Common values can be collected into a *base* dictionary. This
minimizes repetition of identical values in each of the
``lookup_dict`` sub-dictionaries. Now only the values that are
different from the base must be specified of the alternates:
:file:`map.jinja`:
.. code-block:: jinja
{% set mysql = salt['grains.filter_by']({
'default': {
'server': 'mysql-server',
'client': 'mysql-client',
'service': 'mysql',
'config': '/etc/mysql/my.cnf',
'python': 'python-mysqldb',
},
'Debian': {
},
'RedHat': {
'client': 'mysql',
'service': 'mysqld',
'config': '/etc/my.cnf',
'python': 'MySQL-python',
},
'Gentoo': {
'server': 'dev-db/mysql',
'client': 'dev-db/mysql',
'python': 'dev-python/mysql-python',
},
},
merge=salt['pillar.get']('mysql:lookup'),
base=default) %}
Overriding values in the lookup table
`````````````````````````````````````

36
salt/modules/grains.py
View file

@ -409,10 +409,13 @@ def filter_by(lookup_dict, grain='os_family', merge=None, default='default', bas
Python version from the default Python version provided by the OS
(e.g., ``python26-mysql`` instead of ``python-mysql``).
:param default: default lookup_dict's key used if the grain does not exists
or if the grain value has no match on lookup_dict.
:param base: A dictionary to use as a base set of defaults. The grain-selected
``lookup_dict`` is merged over this and then finally the ``merge``
dictionary is merged.
or if the grain value has no match on lookup_dict. If unspecified
the value is "default".
:param base: A lookup_dict key to use for a base dictionary. The
grain-selected ``lookup_dict`` is merged over this and then finally
the ``merge`` dictionary is merged. This allows common values for
each case to be collected in the base and overridden by the grain
selection dictionary and the merge dictionary. Default is unset.
.. versionadded:: 2014.1.0
@ -421,11 +424,13 @@ def filter_by(lookup_dict, grain='os_family', merge=None, default='default', bas
.. code-block:: bash
salt '*' grains.filter_by '{Debian: Debheads rule, RedHat: I love my hat}'
# this one will render {D: {E: I, G: H}, J: K, L: M}
salt '*' grains.filter_by '{A: {B: Z}, C: {D: {E: F,G: H}}}' 'xxx' '{D: {E: I},J: K}' 'C' '{D: {E: X}, L: M}'
# The same with default selected as 'A' rather than 'C':
# {B: Z, D: {E: I}, J: K, L: M}
# this one will render {D: {E: I, G: H}, J: K}
salt '*' grains.filter_by '{A: B, C: {D: {E: F,G: H}}}' 'xxx' '{D: {E: I},J: K}' 'C'
# next one renders {A: {B: G}, D: J}
salt '*' grains.filter_by '{default: {A: {B: C}, D: E}, F: {A: {B: G}}, H: {D: I}}' 'xxx' '{D: J}' 'F' 'default'
# next same as above when default='H' instead of 'F' renders {A: {B: C}, D: J}
'''
ret = lookup_dict.get(
__grains__.get(
grain, default),
@ -433,14 +438,15 @@ def filter_by(lookup_dict, grain='os_family', merge=None, default='default', bas
default, None)
)
if base:
if not isinstance(base, collections.Mapping):
raise SaltException('filter_by base argument must be a dictionary.')
if base and base in lookup_dict:
base_values = lookup_dict[base]
if ret is None:
ret = copy.deepcopy(base)
else:
ret = salt.utils.dictupdate.update(copy.deepcopy(base), ret)
ret = base_values
elif isinstance(base_values, collections.Mapping):
if not isinstance(ret, collections.Mapping):
raise SaltException('filter_by default and look-up values must both be dictionaries.')
ret = salt.utils.dictupdate.update(copy.deepcopy(base_values), ret)
if merge:
if not isinstance(merge, collections.Mapping):

3
salt/modules/mac_user.py
View file

@ -12,6 +12,7 @@ import logging
import random
import string
import time
import itertools
# Import salt libs
import salt.utils
@ -55,7 +56,7 @@ def _dscl(cmd, ctype='create'):
def _first_avail_uid():
uids = set(x.pw_uid for x in pwd.getpwall())
for idx in xrange(501, 2 ** 32):
for idx in itertools.count(501, 2 ** 32):
if idx not in uids:
return idx

98
tests/unit/modules/grains_test.py
View file

@ -1,5 +1,7 @@
# -*- coding: utf-8 -*-
import copy
# Import Salt Testing libs
from salttesting import TestCase
from salttesting.helpers import ensure_in_syspath
@ -9,9 +11,12 @@ ensure_in_syspath('../../')
# Import Salt libs
from salt.exceptions import SaltException
from salt.modules import grains as grainsmod
from salt.utils import dictupdate
grainsmod.__grains__ = {
'os_family': 'MockedOS'
'os_family': 'MockedOS',
'1': '1',
'2': '2',
}
@ -19,7 +24,29 @@ class GrainsModuleTestCase(TestCase):
def test_filter_by(self):
dict1 = {'A': 'B', 'C': {'D': {'E': 'F', 'G': 'H'}}}
mdict = {'D': {'E': 'I'}, 'J': 'K'}
dict2 = {
'default': {
'A': 'B',
'C': {
'D': 'E'
},
},
'1': {
'A': 'X',
},
'2': {
'C': {
'D': 'H',
},
},
'MockedOS': {
'A': 'Z',
},
}
mdict1 = {'D': {'E': 'I'}, 'J': 'K'}
mdict2 = {'A': 'Z'}
mdict3 = {'C': {'D': 'J'}}
# test None result with non existent grain and no default
res = grainsmod.filter_by(dict1, grain='xxx')
@ -34,14 +61,14 @@ class GrainsModuleTestCase(TestCase):
self.assertEqual(res, {'D': {'E': 'F', 'G': 'H'}})
# add a merge dictionary, F disappears
res = grainsmod.filter_by(dict1, grain='xxx', merge=mdict, default='C')
res = grainsmod.filter_by(dict1, grain='xxx', merge=mdict1, default='C')
self.assertEqual(res, {'D': {'E': 'I', 'G': 'H'}, 'J': 'K'})
# dict1 was altered, reestablish
dict1 = {'A': 'B', 'C': {'D': {'E': 'F', 'G': 'H'}}}
# default is not present in dict1, check we only have merge in result
res = grainsmod.filter_by(dict1, grain='xxx', merge=mdict, default='Z')
self.assertEqual(res, mdict)
res = grainsmod.filter_by(dict1, grain='xxx', merge=mdict1, default='Z')
self.assertEqual(res, mdict1)
# default is not present in dict1, and no merge, should get None
res = grainsmod.filter_by(dict1, grain='xxx', default='Z')
@ -62,12 +89,12 @@ class GrainsModuleTestCase(TestCase):
self.assertIs(res, None)
res = grainsmod.filter_by(dict1, default='C')
self.assertEqual(res, {'D': {'E': 'F', 'G': 'H'}})
res = grainsmod.filter_by(dict1, merge=mdict, default='C')
res = grainsmod.filter_by(dict1, merge=mdict1, default='C')
self.assertEqual(res, {'D': {'E': 'I', 'G': 'H'}, 'J': 'K'})
# dict1 was altered, reestablish
dict1 = {'A': 'B', 'C': {'D': {'E': 'F', 'G': 'H'}}}
res = grainsmod.filter_by(dict1, merge=mdict, default='Z')
self.assertEqual(res, mdict)
res = grainsmod.filter_by(dict1, merge=mdict1, default='Z')
self.assertEqual(res, mdict1)
res = grainsmod.filter_by(dict1, default='Z')
self.assertIs(res, None)
# this one is in fact a traceback in updatedict, merging a string with a dictionary
@ -75,7 +102,7 @@ class GrainsModuleTestCase(TestCase):
TypeError,
grainsmod.filter_by,
dict1,
merge=mdict,
merge=mdict1,
default='A'
)
@ -85,17 +112,66 @@ class GrainsModuleTestCase(TestCase):
self.assertEqual(res, {'D': {'E': 'F', 'G': 'H'}})
res = grainsmod.filter_by(dict1, default='A')
self.assertEqual(res, {'D': {'E': 'F', 'G': 'H'}})
res = grainsmod.filter_by(dict1, merge=mdict, default='A')
res = grainsmod.filter_by(dict1, merge=mdict1, default='A')
self.assertEqual(res, {'D': {'E': 'I', 'G': 'H'}, 'J': 'K'})
# dict1 was altered, reestablish
dict1 = {'A': 'B', 'MockedOS': {'D': {'E': 'F', 'G': 'H'}}}
res = grainsmod.filter_by(dict1, merge=mdict, default='Z')
res = grainsmod.filter_by(dict1, merge=mdict1, default='Z')
self.assertEqual(res, {'D': {'E': 'I', 'G': 'H'}, 'J': 'K'})
# dict1 was altered, reestablish
dict1 = {'A': 'B', 'MockedOS': {'D': {'E': 'F', 'G': 'H'}}}
res = grainsmod.filter_by(dict1, default='Z')
self.assertEqual(res, {'D': {'E': 'F', 'G': 'H'}})
# Base tests
# NOTE: these may fail to detect errors if dictupdate.update() is broken
# but then the unit test for dictupdate.update() should fail and expose
# that. The purpose of these tests is it validate the logic of how
# in filter_by() processes its arguments.
# Test with just the base
res = grainsmod.filter_by(dict2, grain='xxx', default='xxx', base='default')
self.assertEqual(res, dict2['default'])
# Test the base with the OS grain look-up
res = grainsmod.filter_by(dict2, default='xxx', base='default')
self.assertEqual(
res,
dictupdate.update(copy.deepcopy(dict2['default']), dict2['MockedOS'])
)
# Test the base with default
res = grainsmod.filter_by(dict2, grain='xxx', base='default')
self.assertEqual(res, dict2['default'])
res = grainsmod.filter_by(dict2, grain='1', base='default')
self.assertEqual(
res,
dictupdate.update(copy.deepcopy(dict2['default']), dict2['1'])
)
res = grainsmod.filter_by(dict2, base='default', merge=mdict2)
self.assertEqual(
res,
dictupdate.update(
dictupdate.update(
copy.deepcopy(dict2['default']),
dict2['MockedOS']),
mdict2
)
)
res = grainsmod.filter_by(dict2, base='default', merge=mdict3)
self.assertEqual(
res,
dictupdate.update(
dictupdate.update(
copy.deepcopy(dict2['default']),
dict2['MockedOS']),
mdict3
)
)
if __name__ == '__main__':
from integration import run_tests

71
tests/unit/utils/dictupdate_test.py Normal file
View file

@ -0,0 +1,71 @@
# -*- coding: utf-8 -*-
import copy
# Import Salt Testing libs
from salttesting import TestCase
from salttesting.helpers import ensure_in_syspath
ensure_in_syspath('../../')
# Import Salt libs
from salt.exceptions import SaltException
from salt.utils import dictupdate
class UtilDictupdateTestCase(TestCase):
dict1 = {'A': 'B', 'C': {'D': 'E', 'F': {'G': 'H', 'I': 'J'}}}
def test_update(self):
# level 1 value changes
mdict = copy.deepcopy(self.dict1)
mdict['A'] = 'Z'
res = dictupdate.update(copy.deepcopy(self.dict1), {'A': 'Z'})
self.assertEqual(res, mdict)
# level 2 value changes
mdict = copy.deepcopy(self.dict1)
mdict['C']['D'] = 'Z'
res = dictupdate.update(copy.deepcopy(self.dict1), {'C': {'D': 'Z'}})
self.assertEqual(res, mdict)
# level 3 value changes
mdict = copy.deepcopy(self.dict1)
mdict['C']['F']['G'] = 'Z'
res = dictupdate.update(
copy.deepcopy(self.dict1),
{'C': {'F': {'G': 'Z'}}}
)
self.assertEqual(res, mdict)
# replace a sub-dictionary
mdict = copy.deepcopy(self.dict1)
mdict['C'] = 'Z'
res = dictupdate.update(copy.deepcopy(self.dict1), {'C': 'Z'})
self.assertEqual(res, mdict)
# add a new scalar value
mdict = copy.deepcopy(self.dict1)
mdict['Z'] = 'Y'
res = dictupdate.update(copy.deepcopy(self.dict1), {'Z': 'Y'})
self.assertEqual(res, mdict)
# add a dictionary
mdict = copy.deepcopy(self.dict1)
mdict['Z'] = {'Y': 'X'}
res = dictupdate.update(copy.deepcopy(self.dict1), {'Z': {'Y': 'X'}})
self.assertEqual(res, mdict)
# add a nested dictionary
mdict = copy.deepcopy(self.dict1)
mdict['Z'] = {'Y': {'X': 'W'}}
res = dictupdate.update(
copy.deepcopy(self.dict1),
{'Z': {'Y': {'X': 'W'}}}
)
self.assertEqual(res, mdict)
if __name__ == '__main__':
from integration import run_tests
run_tests(UtilDictupdateTestCase, needs_daemon=False)