From 3c9130f9f0c04826b28dd42729625e5c4a026188 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Sat, 1 Jul 2017 01:07:54 +0200 Subject: [PATCH 001/159] utils.jinja: use utils.yamldumper for safe yaml dumping --- salt/utils/jinja.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index f122f016510..7152a4d20af 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -22,7 +22,6 @@ from jinja2.environment import TemplateModule from jinja2.ext import Extension from jinja2.exceptions import TemplateRuntimeError import jinja2 -import yaml # Import salt libs import salt @@ -30,6 +29,7 @@ import salt.utils import salt.utils.url import salt.fileclient from salt.utils.odict import OrderedDict +import salt.utils.yamldumper log = logging.getLogger(__name__) @@ -40,18 +40,6 @@ __all__ = [ GLOBAL_UUID = uuid.UUID('91633EBF-1C86-5E33-935A-28061F4B480E') -# To dump OrderedDict objects as regular dicts. Used by the yaml -# template filter. - - -class OrderedDictDumper(yaml.Dumper): # pylint: disable=W0232 - pass - - -yaml.add_representer(OrderedDict, - yaml.representer.SafeRepresenter.represent_dict, - Dumper=OrderedDictDumper) - class SaltCacheLoader(BaseLoader): ''' @@ -717,8 +705,8 @@ class SerializerExtension(Extension, object): return Markup(json.dumps(value, sort_keys=sort_keys, indent=indent).strip()) def format_yaml(self, value, flow_style=True): - yaml_txt = yaml.dump(value, default_flow_style=flow_style, - Dumper=OrderedDictDumper).strip() + yaml_txt = salt.utils.yamldumper.safe_dump( + value, default_flow_style=flow_style).strip() if yaml_txt.endswith('\n...'): yaml_txt = yaml_txt[:len(yaml_txt)-4] return Markup(yaml_txt) From 393fe061b2b96f7d43272999ef086c44774b67e0 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Sat, 1 Jul 2017 02:20:06 +0200 Subject: [PATCH 002/159] jinja utils: yaml import still necessary --- salt/utils/jinja.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index 7152a4d20af..2c23cd8a0f0 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -22,6 +22,7 @@ from jinja2.environment import TemplateModule from jinja2.ext import Extension from jinja2.exceptions import TemplateRuntimeError import jinja2 +import yaml # Import salt libs import salt From 60e001733bd06250c6a91dc4beb6e603e9c30b53 Mon Sep 17 00:00:00 2001 From: Ronald van Zantvoort Date: Thu, 21 Sep 2017 23:11:10 +0200 Subject: [PATCH 003/159] make cached pillars use pillarenv rather than saltenv --- salt/pillar/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py index 8d5eb7e998b..0fbfbfea834 100644 --- a/salt/pillar/__init__.py +++ b/salt/pillar/__init__.py @@ -235,25 +235,25 @@ class PillarCache(object): return fresh_pillar.compile_pillar() # FIXME We are not yet passing pillar_dirs in here def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs - log.debug('Scanning pillar cache for information about minion {0} and saltenv {1}'.format(self.minion_id, self.saltenv)) + log.debug('Scanning pillar cache for information about minion {0} and pillarenv {1}'.format(self.minion_id, self.pillarenv)) log.debug('Scanning cache: {0}'.format(self.cache._dict)) # Check the cache! if self.minion_id in self.cache: # Keyed by minion_id # TODO Compare grains, etc? - if self.saltenv in self.cache[self.minion_id]: + if self.pillarenv in self.cache[self.minion_id]: # We have a cache hit! Send it back. - log.debug('Pillar cache hit for minion {0} and saltenv {1}'.format(self.minion_id, self.saltenv)) - return self.cache[self.minion_id][self.saltenv] + log.debug('Pillar cache hit for minion {0} and pillarenv {1}'.format(self.minion_id, self.pillarenv)) + return self.cache[self.minion_id][self.pillarenv] else: # We found the minion but not the env. Store it. fresh_pillar = self.fetch_pillar() - self.cache[self.minion_id][self.saltenv] = fresh_pillar - log.debug('Pillar cache miss for saltenv {0} for minion {1}'.format(self.saltenv, self.minion_id)) + self.cache[self.minion_id][self.pillarenv] = fresh_pillar + log.debug('Pillar cache miss for pillarenv {0} for minion {1}'.format(self.pillarenv, self.minion_id)) return fresh_pillar else: # We haven't seen this minion yet in the cache. Store it. fresh_pillar = self.fetch_pillar() - self.cache[self.minion_id] = {self.saltenv: fresh_pillar} + self.cache[self.minion_id] = {self.pillarenv: fresh_pillar} log.debug('Pillar cache miss for minion {0}'.format(self.minion_id)) log.debug('Current pillar cache: {0}'.format(self.cache._dict)) # FIXME hack! return fresh_pillar From 765504c1374a3140b32fdeb4bf6ed215268a8be4 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Fri, 22 Sep 2017 09:16:50 +0000 Subject: [PATCH 004/159] Add all the possible keys to the result --- salt/runners/net.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/salt/runners/net.py b/salt/runners/net.py index 86017f4ae1f..2736c9315ee 100644 --- a/salt/runners/net.py +++ b/salt/runners/net.py @@ -812,7 +812,25 @@ def find(addr, best=True, display=_DEFAULT_DISPLAY): ip = '' # pylint: disable=invalid-name ipnet = None - results = {} + results = { + 'int_net': [], + 'int_descr': [], + 'int_name': [], + 'int_ip': [], + 'int_mac': [], + 'int_device': [], + 'lldp_descr': [], + 'lldp_int': [], + 'lldp_device': [], + 'lldp_mac': [], + 'lldp_device_int': [], + 'mac_device': [], + 'mac_int': [], + 'arp_device': [], + 'arp_int': [], + 'arp_mac': [], + 'arp_ip': [] + } if isinstance(addr, int): results['mac'] = findmac(vlan=addr, display=display) From 4b2f791bd2f4b619f6fd8f32e6dd8eb3e64c1ab0 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Fri, 22 Sep 2017 09:17:14 +0000 Subject: [PATCH 005/159] Check if addr is short IPv6 When the addr is sent as a short IPv6 address, e.g., `2804:3be0::`, if might be conflated with the `28:04:3B:E0:00:00` MAC address, which makes the runner look into other data sets than it should. --- salt/runners/net.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/runners/net.py b/salt/runners/net.py index 2736c9315ee..060d44dcabd 100644 --- a/salt/runners/net.py +++ b/salt/runners/net.py @@ -75,6 +75,7 @@ from __future__ import unicode_literals # Import salt lib import salt.output +import salt.utils.network from salt.ext import six from salt.ext.six.moves import map @@ -844,6 +845,8 @@ def find(addr, best=True, display=_DEFAULT_DISPLAY): except IndexError: # no problem, let's keep searching pass + if salt.utils.network.is_ipv6(addr): + mac = False if not mac: try: ip = napalm_helpers.convert(napalm_helpers.ip, addr) # pylint: disable=invalid-name From ab43958926a05d576161bddd2e0378d0461325f9 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 27 Sep 2017 15:27:17 -0700 Subject: [PATCH 006/159] Updating the text that this function is looking for to include the equal sign, in the event that InitiatorName appears somewhere in a comment. --- salt/grains/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 5cdcdccdf5e..232330adcf4 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -2517,7 +2517,7 @@ def _linux_iqn(): if os.path.isfile(initiator): with salt.utils.files.fopen(initiator, 'r') as _iscsi: for line in _iscsi: - if line.find('InitiatorName') != -1: + if line.find('InitiatorName=') != -1: iqn = line.split('=') final_iqn = iqn[1].rstrip() ret.extend([final_iqn]) From 4b1d5f3005bb398e25ded09acaae06a5ae77da56 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 28 Sep 2017 16:53:19 -0700 Subject: [PATCH 007/159] Adding additional tests for grains/isci_iqn --- tests/unit/grains/test_core.py | 49 ++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py index 5fb2d855318..71af36cf1f8 100644 --- a/tests/unit/grains/test_core.py +++ b/tests/unit/grains/test_core.py @@ -464,3 +464,52 @@ PATCHLEVEL = 3 self.assertEqual(os_grains.get('osrelease'), os_release_map['osrelease']) self.assertListEqual(list(os_grains.get('osrelease_info')), os_release_map['osrelease_info']) self.assertEqual(os_grains.get('osmajorrelease'), os_release_map['osmajorrelease']) + + def test_windows_iscsi_iqn_grains(self): + cmd_run_mock = MagicMock( + return_value={'stdout': 'iqn.1991-05.com.microsoft:username-pc.local'} + ) + + with patch.object(salt.utils.platform, 'is_linux', + MagicMock(return_value=False)): + with patch.object(salt.utils.platform, 'is_windows', + MagicMock(return_value=True)): + with patch.dict(core.__salt__, {'run_all': cmd_run_mock}): + with patch.object(salt.utils.path, 'which', + MagicMock(return_value=True)): + with patch.dict(core.__salt__, {'cmd.run_all': cmd_run_mock}): + _grains = core.iscsi_iqn() + + self.assertEqual(_grains.get('iscsi_iqn'), + ['iqn.1991-05.com.microsoft:username-pc.local']) + + def test_aix_iscsi_iqn_grains(self): + cmd_run_mock = MagicMock( + return_value='initiator_name iqn.localhost.hostid.7f000001' + ) + + with patch.object(salt.utils.platform, 'is_linux', + MagicMock(return_value=False)): + with patch.object(salt.utils.platform, 'is_aix', + MagicMock(return_value=True)): + with patch.dict(core.__salt__, {'cmd.run': cmd_run_mock}): + _grains = core.iscsi_iqn() + + self.assertEqual(_grains.get('iscsi_iqn'), + ['iqn.localhost.hostid.7f000001']) + + def test_linux_iscsi_iqn_grains(self): + _iscsi_file = '## DO NOT EDIT OR REMOVE THIS FILE!\n' \ + '## If you remove this file, the iSCSI daemon will not start.\n' \ + '## If you change the InitiatorName, existing access control lists\n' \ + '## may reject this initiator. The InitiatorName must be unique\n' \ + '## for each iSCSI initiator. Do NOT duplicate iSCSI InitiatorNames.\n' \ + 'InitiatorName=iqn.1993-08.org.debian:01:d12f7aba36\n' + + with patch('os.path.isfile', MagicMock(return_value=True)): + with patch('salt.utils.files.fopen', mock_open()) as iscsi_initiator_file: + iscsi_initiator_file.return_value.__iter__.return_value = _iscsi_file.splitlines() + _grains = core.iscsi_iqn() + + self.assertEqual(_grains.get('iscsi_iqn'), + ['iqn.1993-08.org.debian:01:d12f7aba36']) From f06bb04348d04501d3ef68cf0a2fd877bdda6649 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 28 Sep 2017 20:31:08 -0700 Subject: [PATCH 008/159] Adding the right mocked data to the Windows test. Updating the grains/core.py to check if the entire line is alpha_numeric characters to determine if it should be skipped. --- salt/grains/core.py | 2 +- tests/unit/grains/test_core.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 232330adcf4..8ebb656be1d 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -2574,7 +2574,7 @@ def _windows_iqn(): wmic, namespace, mspath, get)) for line in cmdret['stdout'].splitlines(): - if line[0].isalpha(): + if line.isalpha(): continue line = line.rstrip() ret.extend([line]) diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py index 71af36cf1f8..5fd38a820a3 100644 --- a/tests/unit/grains/test_core.py +++ b/tests/unit/grains/test_core.py @@ -467,7 +467,7 @@ PATCHLEVEL = 3 def test_windows_iscsi_iqn_grains(self): cmd_run_mock = MagicMock( - return_value={'stdout': 'iqn.1991-05.com.microsoft:username-pc.local'} + return_value={'stdout': 'iSCSINodeName\niqn.1991-05.com.microsoft:simon-x1\n'} ) with patch.object(salt.utils.platform, 'is_linux', @@ -481,7 +481,7 @@ PATCHLEVEL = 3 _grains = core.iscsi_iqn() self.assertEqual(_grains.get('iscsi_iqn'), - ['iqn.1991-05.com.microsoft:username-pc.local']) + ['iqn.1991-05.com.microsoft:simon-x1']) def test_aix_iscsi_iqn_grains(self): cmd_run_mock = MagicMock( From c3fb944e11c3275064326e37b294f3d0c6ab261a Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 29 Sep 2017 08:08:24 -0700 Subject: [PATCH 009/159] fixing merge conflict. --- salt/grains/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 8ebb656be1d..7c49550a466 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -2578,7 +2578,6 @@ def _windows_iqn(): continue line = line.rstrip() ret.extend([line]) - return ret From 76662b53a85807c6909aeff8194ac08de7ab4eee Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Fri, 29 Sep 2017 14:35:44 -0700 Subject: [PATCH 010/159] Adding commit b097f73 back in. --- salt/grains/core.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 7c49550a466..42813df0491 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -2574,10 +2574,9 @@ def _windows_iqn(): wmic, namespace, mspath, get)) for line in cmdret['stdout'].splitlines(): - if line.isalpha(): - continue - line = line.rstrip() - ret.extend([line]) + if line.startswith('iqn.'): + line = line.rstrip() + ret.extend([line]) return ret From cc61e7f63802c620b256e913dcde000c45cfe85d Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Sat, 30 Sep 2017 13:16:40 -0700 Subject: [PATCH 011/159] swapping out line.find for line.startswith. --- salt/grains/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index 42813df0491..a949a09a87f 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -2517,7 +2517,7 @@ def _linux_iqn(): if os.path.isfile(initiator): with salt.utils.files.fopen(initiator, 'r') as _iscsi: for line in _iscsi: - if line.find('InitiatorName=') != -1: + if line.starswith('InitiatorName='): iqn = line.split('=') final_iqn = iqn[1].rstrip() ret.extend([final_iqn]) From 4cb86e5f3d2d4753e601b3fbef944970b1d9125c Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Sat, 30 Sep 2017 16:59:48 -0700 Subject: [PATCH 012/159] Fixing a typo. --- salt/grains/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index a949a09a87f..6c7aed03439 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -2517,7 +2517,7 @@ def _linux_iqn(): if os.path.isfile(initiator): with salt.utils.files.fopen(initiator, 'r') as _iscsi: for line in _iscsi: - if line.starswith('InitiatorName='): + if line.startswith('InitiatorName='): iqn = line.split('=') final_iqn = iqn[1].rstrip() ret.extend([final_iqn]) From 9572aabb6781f3a4247bfb4d3d33ac1457db7600 Mon Sep 17 00:00:00 2001 From: Morgan Willcock Date: Thu, 26 Oct 2017 19:31:56 +0100 Subject: [PATCH 013/159] Fix traceback and incorrect message when resolving an unresolvable SID --- salt/utils/win_dacl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/utils/win_dacl.py b/salt/utils/win_dacl.py index 749cb1665d3..923c1659756 100644 --- a/salt/utils/win_dacl.py +++ b/salt/utils/win_dacl.py @@ -1133,7 +1133,7 @@ def get_name(principal): try: return win32security.LookupAccountSid(None, sid_obj)[0] - except TypeError: + except (pywintypes.error, TypeError): raise CommandExecutionError( 'Could not find User for {0}'.format(principal)) @@ -1173,7 +1173,7 @@ def get_owner(obj_name): owner_sid = 'S-1-1-0' else: raise CommandExecutionError( - 'Failed to set permissions: {0}'.format(exc.strerror)) + 'Failed to get owner: {0}'.format(exc.strerror)) return get_name(win32security.ConvertSidToStringSid(owner_sid)) From 4f382df5c7397d962f5302b9aa38286d8b06fc05 Mon Sep 17 00:00:00 2001 From: Super-User Date: Wed, 1 Nov 2017 13:28:35 +0100 Subject: [PATCH 014/159] Add support to zpool for pausing scrub. This adds support for the -p flag introduce in https://github.com/openzfs/openzfs/pull/407 --- salt/modules/zpool.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/salt/modules/zpool.py b/salt/modules/zpool.py index 9433c87abd3..7a3c0f3da39 100644 --- a/salt/modules/zpool.py +++ b/salt/modules/zpool.py @@ -527,7 +527,7 @@ def destroy(zpool, force=False): return ret -def scrub(zpool, stop=False): +def scrub(zpool, stop=False, pause=False): ''' .. versionchanged:: 2016.3.0 @@ -537,6 +537,13 @@ def scrub(zpool, stop=False): name of storage pool stop : boolean if true, cancel ongoing scrub + pause : boolean + if true, pause ongoing scrub + .. versionadded:: Oxygen + + .. note:: + + If both pause and stop are true, stop will win. CLI Example: @@ -548,9 +555,16 @@ def scrub(zpool, stop=False): ret[zpool] = {} if exists(zpool): zpool_cmd = _check_zpool() - cmd = '{zpool_cmd} scrub {stop}{zpool}'.format( + if stop: + action = '-s ' + elif pause: + # NOTE: https://github.com/openzfs/openzfs/pull/407 + action = '-p ' + else: + action = '' + cmd = '{zpool_cmd} scrub {action}{zpool}'.format( zpool_cmd=zpool_cmd, - stop='-s ' if stop else '', + action=action, zpool=zpool ) res = __salt__['cmd.run_all'](cmd, python_shell=False) @@ -565,7 +579,12 @@ def scrub(zpool, stop=False): else: ret[zpool]['error'] = res['stdout'] else: - ret[zpool]['scrubbing'] = True if not stop else False + if stop: + ret[zpool]['scrubbing'] = False + elif pause: + ret[zpool]['scrubbing'] = False + else: + ret[zpool]['scrubbing'] = True else: ret[zpool] = 'storage pool does not exist' From 8e8d5c13470476952539df13756ea2d67884435f Mon Sep 17 00:00:00 2001 From: Super-User Date: Wed, 1 Nov 2017 14:09:35 +0100 Subject: [PATCH 015/159] Add boot part creation support to zpool.create --- salt/modules/zpool.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/salt/modules/zpool.py b/salt/modules/zpool.py index 7a3c0f3da39..92374bb91c6 100644 --- a/salt/modules/zpool.py +++ b/salt/modules/zpool.py @@ -612,6 +612,9 @@ def create(zpool, *vdevs, **kwargs): additional pool properties filesystem_properties : dict additional filesystem properties + createboot : boolean + ..versionadded:: Oxygen + create a boot partition CLI Example: @@ -658,14 +661,21 @@ def create(zpool, *vdevs, **kwargs): zpool_cmd = _check_zpool() force = kwargs.get('force', False) altroot = kwargs.get('altroot', None) + createboot = kwargs.get('createboot', False) mountpoint = kwargs.get('mountpoint', None) properties = kwargs.get('properties', None) filesystem_properties = kwargs.get('filesystem_properties', None) cmd = '{0} create'.format(zpool_cmd) + # bootsize implies createboot + if properties and 'bootsize' in properties: + createboot = True + # apply extra arguments from kwargs if force: # force creation cmd = '{0} -f'.format(cmd) + if createboot: # create boot paritition + cmd = '{0} -B'.format(cmd) if properties: # create "-o property=value" pairs optlist = [] for prop in properties: From 6683250538a3bfca929c18bc8427ea54ef540713 Mon Sep 17 00:00:00 2001 From: Super-User Date: Wed, 1 Nov 2017 14:30:04 +0100 Subject: [PATCH 016/159] Add support for zpool.labelclear --- salt/modules/zpool.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/salt/modules/zpool.py b/salt/modules/zpool.py index 92374bb91c6..9686af92286 100644 --- a/salt/modules/zpool.py +++ b/salt/modules/zpool.py @@ -1252,6 +1252,44 @@ def offline(zpool, *vdevs, **kwargs): return ret +def labelclear(device, force=False): + ''' + .. versionadded:: Oxygen + + Removes ZFS label information from the specified device + + .. warning:: + + The device must not be part of an active pool configuration. + + device : string + device + force : boolean + treat exported or foreign devices as inactive + + CLI Example: + + .. code-block:: bash + + salt '*' zpool.labelclear /path/to/dev + ''' + ret = {} + + zpool_cmd = _check_zpool() + cmd = '{zpool_cmd} labelclear {force}{device}'.format( + zpool_cmd=zpool_cmd, + force='-f ' if force else '', + device=device, + ) + # Bring all specified devices offline + res = __salt__['cmd.run_all'](cmd, python_shell=False) + if res['retcode'] != 0: + ret[device] = res['stderr'] if 'stderr' in res else res['stdout'] + else: + ret[device] = 'cleared' + return ret + + def reguid(zpool): ''' .. versionadded:: 2016.3.0 From a04fb62f48aa27050b6ebe4f52ea89397c8437ce Mon Sep 17 00:00:00 2001 From: Super-User Date: Wed, 1 Nov 2017 14:52:13 +0100 Subject: [PATCH 017/159] Add support for zpool.split --- salt/modules/zpool.py | 89 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/salt/modules/zpool.py b/salt/modules/zpool.py index 9686af92286..b9f1616233d 100644 --- a/salt/modules/zpool.py +++ b/salt/modules/zpool.py @@ -875,6 +875,95 @@ def detach(zpool, device): return ret +def split(zpool, newzpool, **kwargs): + ''' + .. versionadded:: Oxygen + + Splits devices off pool creating newpool. + + .. note:: + + All vdevs in pool must be mirrors. At the time of the split, + newpool will be a replica of pool. + + zpool : string + name of storage pool + newzpool : string + name of new storage pool + mountpoint : string + sets the mount point for the root dataset + altroot : string + sets altroot for newzpool + properties : dict + additional pool properties for newzpool + + CLI Example: + + .. code-block:: bash + + salt '*' zpool.split datamirror databackup + salt '*' zpool.split datamirror databackup altroot=/backup + + .. note:: + + Zpool properties can be specified at the time of creation of the pool by + passing an additional argument called "properties" and specifying the properties + with their respective values in the form of a python dictionary:: + + properties="{'property1': 'value1', 'property2': 'value2'}" + + Example: + + .. code-block:: bash + + salt '*' zpool.split datamirror databackup properties="{'readonly': 'on'}" + ''' + ret = {} + + # Check if the pool_name is already being used + if exists(newzpool): + ret[newzpool] = 'storage pool already exists' + return ret + + if not exists(zpool): + ret[zpool] = 'storage pool does not exists' + return ret + + zpool_cmd = _check_zpool() + altroot = kwargs.get('altroot', None) + properties = kwargs.get('properties', None) + cmd = '{0} split'.format(zpool_cmd) + + # apply extra arguments from kwargs + if properties: # create "-o property=value" pairs + optlist = [] + for prop in properties: + if isinstance(properties[prop], bool): + value = 'on' if properties[prop] else 'off' + else: + if ' ' in properties[prop]: + value = "'{0}'".format(properties[prop]) + else: + value = properties[prop] + optlist.append('-o {0}={1}'.format(prop, value)) + opts = ' '.join(optlist) + cmd = '{0} {1}'.format(cmd, opts) + if altroot: # set altroot + cmd = '{0} -R {1}'.format(cmd, altroot) + cmd = '{0} {1} {2}'.format(cmd, zpool, newzpool) + + # Create storage pool + res = __salt__['cmd.run_all'](cmd, python_shell=False) + + # Check and see if the pools is available + if res['retcode'] != 0: + ret[newzpool] = res['stderr'] if 'stderr' in res else res['stdout'] + else: + ret[newzpool] = 'split off from {}'.format(zpool) + + return ret + + def replace(zpool, old_device, new_device=None, force=False): ''' .. versionchanged:: 2016.3.0 From 52f73835b8378672c1e8a24730b8e8bf461f13b1 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Tue, 7 Nov 2017 16:39:57 +0000 Subject: [PATCH 018/159] Adapt napalm modules to the new library structure Starting with November 2017, the NAPALM library has been reunified into a monolithic package. This falls with a couple of changes into the Salt codebase. While the users can continue using napalm_base, it is recommended to upgrade to napalm 2.0. At the same time, back in time, the napalm package used to have a different meaning, so we need to check if the release version is at least 2.0. --- salt/proxy/napalm.py | 14 ++------------ salt/utils/napalm.py | 22 ++++++++++++++++++++-- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/salt/proxy/napalm.py b/salt/proxy/napalm.py index 1acee331486..c49651972ac 100644 --- a/salt/proxy/napalm.py +++ b/salt/proxy/napalm.py @@ -129,17 +129,7 @@ from __future__ import absolute_import import logging log = logging.getLogger(__file__) -# Import third party lib -try: - # will try to import NAPALM - # https://github.com/napalm-automation/napalm - # pylint: disable=W0611 - import napalm_base - # pylint: enable=W0611 - HAS_NAPALM = True -except ImportError: - HAS_NAPALM = False - +# Import Salt modules from salt.ext import six import salt.utils.napalm @@ -163,7 +153,7 @@ DETAILS = {} def __virtual__(): - return HAS_NAPALM or (False, 'Please install the NAPALM library: `pip install napalm`!') + return salt.utils.napalm.virtual(__opts__, 'napalm', __file__) # ---------------------------------------------------------------------------------------------------------------------- # helper functions -- will not be exported diff --git a/salt/utils/napalm.py b/salt/utils/napalm.py index 0523e0a568f..5597f991467 100644 --- a/salt/utils/napalm.py +++ b/salt/utils/napalm.py @@ -31,11 +31,27 @@ try: # will try to import NAPALM # https://github.com/napalm-automation/napalm # pylint: disable=W0611 - import napalm_base + import napalm + import napalm.base as napalm_base # pylint: enable=W0611 HAS_NAPALM = True + HAS_NAPALM_BASE = False # doesn't matter anymore, but needed for the logic below + log.debug('napalm seems to be installed') + try: + NAPALM_MAJOR = int(napalm.__version__.split('.')[0]) + log.debug('napalm version: %s', napalm.__version__) + except AttributeError: + NAPALM_MAJOR = 0 except ImportError: + log.info('napalm doesnt seem to be installed, trying to import napalm_base') HAS_NAPALM = False + try: + import napalm_base + log.debug('napalm_base seems to be installed') + HAS_NAPALM_BASE = True + except ImportError: + log.info('napalm_base doesnt seem to be installed either') + HAS_NAPALM_BASE = False try: # try importing ConnectionClosedException @@ -81,7 +97,9 @@ def virtual(opts, virtualname, filename): ''' Returns the __virtual__. ''' - if HAS_NAPALM and (is_proxy(opts) or is_minion(opts)): + if ( (HAS_NAPALM and NAPALM_MAJOR >= 2) or HAS_NAPALM_BASE ) and ( is_proxy(opts) or is_minion(opts) ): + if HAS_NAPALM_BASE: + log.info('You still seem to use napalm_base. Please consider upgrading to napalm >= 2.0.0') return virtualname else: return ( From c6a38258a3f3f1a7758de09ef1c60161c861b8bf Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 8 Nov 2017 15:15:26 +0000 Subject: [PATCH 019/159] Add napalm>2.0.0 note and update URLs --- salt/proxy/napalm.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/salt/proxy/napalm.py b/salt/proxy/napalm.py index c49651972ac..8a5340180eb 100644 --- a/salt/proxy/napalm.py +++ b/salt/proxy/napalm.py @@ -16,9 +16,21 @@ Dependencies The ``napalm`` proxy module requires NAPALM_ library to be installed: ``pip install napalm`` Please check Installation_ for complete details. -.. _NAPALM: https://napalm.readthedocs.io -.. _Installation: https://napalm.readthedocs.io/en/latest/installation.html +.. _NAPALM: https://napalm-automation.net/ +.. _Installation: http://napalm.readthedocs.io/en/latest/installation/index.html +.. note:: + + Beginning with Salt release 2017.7.3, it is recommended to use + ``napalm`` >= ``2.0.0``. The library has been unified into a monolithic + package, as in opposite to separate packages per driver. For more details + you can check `this document `_. + While it will still work with the old packages, bear in mind that the NAPALM + core team will maintain only the main ``napalm`` package. + + Moreover, for additional capabilities, the users can always define a + library that extends NAPALM's base capabilities and configure the + ``provider`` option (see below). Pillar ------ @@ -59,7 +71,7 @@ always_alive: ``True`` .. versionadded:: 2017.7.0 provider: ``napalm_base`` - The module that provides the ``get_network_device`` function. + The library that provides the ``get_network_device`` function. This option is useful when the user has more specific needs and requires to extend the NAPALM capabilities using a private library implementation. The only constraint is that the alternative library needs to have the From 4ef1e3eb97f2f5b3c3961d0d455f1fb380b7be3c Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 29 Sep 2017 12:11:56 -0600 Subject: [PATCH 020/159] Fix `unit.states.test_archive` for Windows Use os agnostic paths Mock `salt.utils.which` for Windows Handle urlparse return on Windows --- salt/states/archive.py | 16 ++++++++++--- tests/unit/states/test_archive.py | 38 +++++++++++++++++++------------ 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/salt/states/archive.py b/salt/states/archive.py index 9fd14d49d8a..b2b3db92360 100644 --- a/salt/states/archive.py +++ b/salt/states/archive.py @@ -765,12 +765,22 @@ def extracted(name, return ret urlparsed_source = _urlparse(source_match) - source_hash_basename = urlparsed_source.path or urlparsed_source.netloc + urlparsed_scheme = urlparsed_source.scheme + urlparsed_path = os.path.join( + urlparsed_source.netloc, + urlparsed_source.path).rstrip(os.sep) - source_is_local = urlparsed_source.scheme in salt.utils.files.LOCAL_PROTOS + if urlparsed_scheme and \ + urlparsed_scheme.lower() in 'abcdefghijklmnopqrstuvwxyz': + urlparsed_path = ':'.join([urlparsed_scheme, urlparsed_path]) + urlparsed_scheme = 'file' + + source_hash_basename = urlparsed_path or urlparsed_source.netloc + + source_is_local = urlparsed_scheme in salt.utils.files.LOCAL_PROTOS if source_is_local: # Get rid of "file://" from start of source_match - source_match = os.path.realpath(os.path.expanduser(urlparsed_source.path)) + source_match = os.path.realpath(os.path.expanduser(urlparsed_path)) if not os.path.isfile(source_match): ret['comment'] = 'Source file \'{0}\' does not exist'.format( salt.utils.url.redact_http_basic_auth(source_match)) diff --git a/tests/unit/states/test_archive.py b/tests/unit/states/test_archive.py index 30d256773d6..e35c01bc195 100644 --- a/tests/unit/states/test_archive.py +++ b/tests/unit/states/test_archive.py @@ -20,6 +20,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.states.archive as archive from salt.ext.six.moves import zip # pylint: disable=import-error,redefined-builtin +import salt.utils def _isfile_side_effect(path): @@ -32,11 +33,14 @@ def _isfile_side_effect(path): If so, just add an entry in the dictionary for the path being used for tar. ''' return { - '/tmp/foo.tar.gz': True, - '/tmp/out': False, - '/usr/bin/tar': True, - '/bin/tar': True, - '/tmp/test_extracted_tar': False, + os.sep + os.path.join('tmp', 'foo.tar.gz'): True, + os.path.join('c:\\tmp', 'foo.tar.gz'): True, + os.sep + os.path.join('tmp', 'out'): False, + os.path.join('c:\\tmp', 'out'): False, + os.sep + os.path.join('usr', 'bin', 'tar'): True, + os.sep + os.path.join('bin', 'tar'): True, + os.sep + os.path.join('tmp', 'test_extracted_tar'): False, + os.path.join('c:\\tmp', 'test_extracted_tar'): False, }[path] @@ -59,8 +63,11 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): archive.extracted tar options ''' - source = '/tmp/foo.tar.gz' - tmp_dir = '/tmp/test_extracted_tar' + source = os.sep + os.path.join('tmp', 'foo.tar.gz') + tmp_dir = os.sep + os.path.join('tmp', 'test_extracted_tar') + if salt.utils.is_windows(): + source = os.path.join('c:\\tmp', 'foo.tar.gz') + tmp_dir = os.path.join('c:\\tmp', 'test_extracted_tar') test_tar_opts = [ '--no-anchored foo', 'v -p --opt', @@ -103,7 +110,8 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'archive.list': list_mock, 'file.source_list': mock_source_list}): with patch.dict(archive.__states__, {'file.directory': mock_true}): - with patch.object(os.path, 'isfile', isfile_mock): + with patch.object(os.path, 'isfile', isfile_mock), \ + patch('salt.utils.which', MagicMock(return_value=True)): for test_opts, ret_opts in zip(test_tar_opts, ret_tar_opts): ret = archive.extracted(tmp_dir, source, @@ -119,7 +127,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): Tests the call of extraction with gnutar ''' gnutar = MagicMock(return_value='tar (GNU tar)') - source = '/tmp/foo.tar.gz' + source = os.sep + os.path.join('tmp', 'foo.tar.gz') mock_false = MagicMock(return_value=False) mock_true = MagicMock(return_value=True) state_single_mock = MagicMock(return_value={'local': {'result': True}}) @@ -144,8 +152,9 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'archive.list': list_mock, 'file.source_list': mock_source_list}): with patch.dict(archive.__states__, {'file.directory': mock_true}): - with patch.object(os.path, 'isfile', isfile_mock): - ret = archive.extracted('/tmp/out', + with patch.object(os.path, 'isfile', isfile_mock), \ + patch('salt.utils.which', MagicMock(return_value=True)): + ret = archive.extracted(os.path.join(os.sep + 'tmp', 'out'), source, options='xvzf', enforce_toplevel=False, @@ -157,7 +166,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): Tests the call of extraction with bsdtar ''' bsdtar = MagicMock(return_value='tar (bsdtar)') - source = '/tmp/foo.tar.gz' + source = os.sep + os.path.join('tmp', 'foo.tar.gz') mock_false = MagicMock(return_value=False) mock_true = MagicMock(return_value=True) state_single_mock = MagicMock(return_value={'local': {'result': True}}) @@ -182,8 +191,9 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'archive.list': list_mock, 'file.source_list': mock_source_list}): with patch.dict(archive.__states__, {'file.directory': mock_true}): - with patch.object(os.path, 'isfile', isfile_mock): - ret = archive.extracted('/tmp/out', + with patch.object(os.path, 'isfile', isfile_mock), \ + patch('salt.utils.which', MagicMock(return_value=True)): + ret = archive.extracted(os.path.join(os.sep + 'tmp', 'out'), source, options='xvzf', enforce_toplevel=False, From ba2f2eb788b17b97c9d24c0877b0b87f1aabcc04 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 23 Oct 2017 11:39:12 -0600 Subject: [PATCH 021/159] Add Erik's changes --- salt/states/archive.py | 5 ++++- tests/unit/states/test_archive.py | 29 +++++++++++++++-------------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/salt/states/archive.py b/salt/states/archive.py index b2b3db92360..025ccf43bf3 100644 --- a/salt/states/archive.py +++ b/salt/states/archive.py @@ -13,6 +13,7 @@ import os import re import shlex import stat +import string import tarfile from contextlib import closing @@ -770,8 +771,10 @@ def extracted(name, urlparsed_source.netloc, urlparsed_source.path).rstrip(os.sep) + # urlparsed_scheme will be the drive letter if this is a Windows file path + # This checks for a drive letter as the scheme and changes it to file if urlparsed_scheme and \ - urlparsed_scheme.lower() in 'abcdefghijklmnopqrstuvwxyz': + urlparsed_scheme.lower() in string.ascii_lowercase: urlparsed_path = ':'.join([urlparsed_scheme, urlparsed_path]) urlparsed_scheme = 'file' diff --git a/tests/unit/states/test_archive.py b/tests/unit/states/test_archive.py index e35c01bc195..c90207c1de9 100644 --- a/tests/unit/states/test_archive.py +++ b/tests/unit/states/test_archive.py @@ -33,14 +33,14 @@ def _isfile_side_effect(path): If so, just add an entry in the dictionary for the path being used for tar. ''' return { - os.sep + os.path.join('tmp', 'foo.tar.gz'): True, - os.path.join('c:\\tmp', 'foo.tar.gz'): True, - os.sep + os.path.join('tmp', 'out'): False, - os.path.join('c:\\tmp', 'out'): False, - os.sep + os.path.join('usr', 'bin', 'tar'): True, - os.sep + os.path.join('bin', 'tar'): True, - os.sep + os.path.join('tmp', 'test_extracted_tar'): False, - os.path.join('c:\\tmp', 'test_extracted_tar'): False, + '/tmp/foo.tar.gz': True, + 'c:\\tmp\\foo.tar.gz': True, + '/tmp/out': False, + '\\tmp\\out': False, + '/usr/bin/tar': True, + '/bin/tar': True, + '/tmp/test_extracted_tar': False, + 'c:\\tmp\\test_extracted_tar': False, }[path] @@ -63,11 +63,12 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): archive.extracted tar options ''' - source = os.sep + os.path.join('tmp', 'foo.tar.gz') - tmp_dir = os.sep + os.path.join('tmp', 'test_extracted_tar') if salt.utils.is_windows(): - source = os.path.join('c:\\tmp', 'foo.tar.gz') - tmp_dir = os.path.join('c:\\tmp', 'test_extracted_tar') + source = 'c:\\tmp\\foo.tar.gz' + tmp_dir = 'c:\\tmp\\test_extracted_tar' + else: + source = '/tmp/foo.tar.gz' + tmp_dir = '/tmp/test_extracted_tar' test_tar_opts = [ '--no-anchored foo', 'v -p --opt', @@ -127,7 +128,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): Tests the call of extraction with gnutar ''' gnutar = MagicMock(return_value='tar (GNU tar)') - source = os.sep + os.path.join('tmp', 'foo.tar.gz') + source = '/tmp/foo.tar.gz' mock_false = MagicMock(return_value=False) mock_true = MagicMock(return_value=True) state_single_mock = MagicMock(return_value={'local': {'result': True}}) @@ -166,7 +167,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): Tests the call of extraction with bsdtar ''' bsdtar = MagicMock(return_value='tar (bsdtar)') - source = os.sep + os.path.join('tmp', 'foo.tar.gz') + source = '/tmp/foo.tar.gz' mock_false = MagicMock(return_value=False) mock_true = MagicMock(return_value=True) state_single_mock = MagicMock(return_value={'local': {'result': True}}) From b1dfe9c3c8683e7250936733e2fe02b88d894b1c Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 9 Nov 2017 14:31:38 -0700 Subject: [PATCH 022/159] Format patching with statements for easier reading --- tests/unit/states/test_archive.py | 78 +++++++++++++++---------------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/tests/unit/states/test_archive.py b/tests/unit/states/test_archive.py index c90207c1de9..f1d3999a59a 100644 --- a/tests/unit/states/test_archive.py +++ b/tests/unit/states/test_archive.py @@ -102,26 +102,24 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(archive.__opts__, {'test': False, 'cachedir': tmp_dir, - 'hash_type': 'sha256'}): - with patch.dict(archive.__salt__, {'file.directory_exists': mock_false, - 'file.file_exists': mock_false, - 'state.single': state_single_mock, - 'file.makedirs': mock_true, - 'cmd.run_all': mock_run, - 'archive.list': list_mock, - 'file.source_list': mock_source_list}): - with patch.dict(archive.__states__, {'file.directory': mock_true}): - with patch.object(os.path, 'isfile', isfile_mock), \ - patch('salt.utils.which', MagicMock(return_value=True)): - for test_opts, ret_opts in zip(test_tar_opts, ret_tar_opts): - ret = archive.extracted(tmp_dir, - source, - options=test_opts, - enforce_toplevel=False) - ret_opts.append(source) - mock_run.assert_called_with(ret_opts, - cwd=tmp_dir + os.sep, - python_shell=False) + 'hash_type': 'sha256'}),\ + patch.dict(archive.__salt__, {'file.directory_exists': mock_false, + 'file.file_exists': mock_false, + 'state.single': state_single_mock, + 'file.makedirs': mock_true, + 'cmd.run_all': mock_run, + 'archive.list': list_mock, + 'file.source_list': mock_source_list}),\ + patch.dict(archive.__states__, {'file.directory': mock_true}),\ + patch.object(os.path, 'isfile', isfile_mock),\ + patch('salt.utils.which', MagicMock(return_value=True)): + + for test_opts, ret_opts in zip(test_tar_opts, ret_tar_opts): + archive.extracted(tmp_dir, source, options=test_opts, + enforce_toplevel=False) + ret_opts.append(source) + mock_run.assert_called_with(ret_opts, cwd=tmp_dir + os.sep, + python_shell=False) def test_tar_gnutar(self): ''' @@ -151,16 +149,16 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'file.makedirs': mock_true, 'cmd.run_all': run_all, 'archive.list': list_mock, - 'file.source_list': mock_source_list}): - with patch.dict(archive.__states__, {'file.directory': mock_true}): - with patch.object(os.path, 'isfile', isfile_mock), \ - patch('salt.utils.which', MagicMock(return_value=True)): - ret = archive.extracted(os.path.join(os.sep + 'tmp', 'out'), - source, - options='xvzf', - enforce_toplevel=False, - keep=True) - self.assertEqual(ret['changes']['extracted_files'], 'stdout') + 'file.source_list': mock_source_list}),\ + patch.dict(archive.__states__, {'file.directory': mock_true}),\ + patch.object(os.path, 'isfile', isfile_mock),\ + patch('salt.utils.which', MagicMock(return_value=True)): + ret = archive.extracted(os.path.join(os.sep + 'tmp', 'out'), + source, + options='xvzf', + enforce_toplevel=False, + keep=True) + self.assertEqual(ret['changes']['extracted_files'], 'stdout') def test_tar_bsdtar(self): ''' @@ -190,13 +188,13 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'file.makedirs': mock_true, 'cmd.run_all': run_all, 'archive.list': list_mock, - 'file.source_list': mock_source_list}): - with patch.dict(archive.__states__, {'file.directory': mock_true}): - with patch.object(os.path, 'isfile', isfile_mock), \ - patch('salt.utils.which', MagicMock(return_value=True)): - ret = archive.extracted(os.path.join(os.sep + 'tmp', 'out'), - source, - options='xvzf', - enforce_toplevel=False, - keep=True) - self.assertEqual(ret['changes']['extracted_files'], 'stderr') + 'file.source_list': mock_source_list}),\ + patch.dict(archive.__states__, {'file.directory': mock_true}),\ + patch.object(os.path, 'isfile', isfile_mock),\ + patch('salt.utils.which', MagicMock(return_value=True)): + ret = archive.extracted(os.path.join(os.sep + 'tmp', 'out'), + source, + options='xvzf', + enforce_toplevel=False, + keep=True) + self.assertEqual(ret['changes']['extracted_files'], 'stderr') From 8c107873cd057868e656216a6437f85989fadd60 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 9 Nov 2017 09:42:03 -0700 Subject: [PATCH 023/159] Remove wmi monkeypatching The WMI object was being monkey patched in the global namespace causing other tests to fail --- salt/modules/win_status.py | 3 +- tests/unit/modules/test_win_dns_client.py | 59 ++++++++++++----------- tests/unit/modules/test_win_network.py | 13 +++-- tests/unit/modules/test_win_status.py | 23 +++------ 4 files changed, 50 insertions(+), 48 deletions(-) diff --git a/salt/modules/win_status.py b/salt/modules/win_status.py index 7197152a066..3f6d8cb3a00 100644 --- a/salt/modules/win_status.py +++ b/salt/modules/win_status.py @@ -6,8 +6,7 @@ or for problem solving if your minion is having problems. .. versionadded:: 0.12.0 -:depends: - pythoncom - - wmi +:depends: - wmi ''' # Import Python Libs diff --git a/tests/unit/modules/test_win_dns_client.py b/tests/unit/modules/test_win_dns_client.py index b02ad75e5e2..9671910fb79 100644 --- a/tests/unit/modules/test_win_dns_client.py +++ b/tests/unit/modules/test_win_dns_client.py @@ -21,6 +21,12 @@ from tests.support.mock import ( # Import Salt Libs import salt.modules.win_dns_client as win_dns_client +try: + import wmi + HAS_WMI = True +except ImportError: + HAS_WMI = False + class Mockwmi(object): ''' @@ -59,6 +65,7 @@ class Mockwinapi(object): @skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_WMI, 'WMI only available on Windows') class WinDnsClientTestCase(TestCase, LoaderModuleMockMixin): ''' Test cases for salt.modules.win_dns_client @@ -66,16 +73,13 @@ class WinDnsClientTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): # wmi and pythoncom modules are platform specific... - wmi = types.ModuleType('wmi') - pythoncom = types.ModuleType('pythoncom') - sys_modules_patcher = patch.dict('sys.modules', {'wmi': wmi, 'pythoncom': pythoncom}) + mock_pythoncom = types.ModuleType('pythoncom') + sys_modules_patcher = patch.dict('sys.modules', + {'pythoncom': mock_pythoncom}) sys_modules_patcher.start() self.addCleanup(sys_modules_patcher.stop) self.WMI = Mock() self.addCleanup(delattr, self, 'WMI') - wmi.WMI = Mock(return_value=self.WMI) - pythoncom.CoInitialize = Mock() - pythoncom.CoUninitialize = Mock() return {win_dns_client: {'wmi': wmi}} # 'get_dns_servers' function tests: 1 @@ -90,7 +94,8 @@ class WinDnsClientTestCase(TestCase, LoaderModuleMockMixin): patch.object(self.WMI, 'Win32_NetworkAdapter', return_value=[Mockwmi()]), \ patch.object(self.WMI, 'Win32_NetworkAdapterConfiguration', - return_value=[Mockwmi()]): + return_value=[Mockwmi()]), \ + patch.object(wmi, 'WMI', Mock(return_value=self.WMI)): self.assertListEqual(win_dns_client.get_dns_servers ('Local Area Connection'), ['10.1.1.10']) @@ -113,23 +118,22 @@ class WinDnsClientTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it add the DNS server to the network interface. ''' - with patch('salt.utils.winapi.Com', MagicMock()): - with patch.object(self.WMI, 'Win32_NetworkAdapter', - return_value=[Mockwmi()]): - with patch.object(self.WMI, 'Win32_NetworkAdapterConfiguration', - return_value=[Mockwmi()]): - self.assertFalse(win_dns_client.add_dns('10.1.1.10', - 'Ethernet')) + with patch('salt.utils.winapi.Com', MagicMock()), \ + patch.object(self.WMI, 'Win32_NetworkAdapter', + return_value=[Mockwmi()]), \ + patch.object(self.WMI, 'Win32_NetworkAdapterConfiguration', + return_value=[Mockwmi()]), \ + patch.object(wmi, 'WMI', Mock(return_value=self.WMI)): + self.assertFalse(win_dns_client.add_dns('10.1.1.10', 'Ethernet')) - self.assertTrue(win_dns_client.add_dns - ('10.1.1.10', 'Local Area Connection')) + self.assertTrue(win_dns_client.add_dns('10.1.1.10', 'Local Area Connection')) with patch.object(win_dns_client, 'get_dns_servers', - MagicMock(return_value=['10.1.1.10'])): - with patch.dict(win_dns_client.__salt__, - {'cmd.retcode': MagicMock(return_value=0)}): - self.assertTrue(win_dns_client.add_dns('10.1.1.0', - 'Local Area Connection')) + MagicMock(return_value=['10.1.1.10'])), \ + patch.dict(win_dns_client.__salt__, + {'cmd.retcode': MagicMock(return_value=0)}), \ + patch.object(wmi, 'WMI', Mock(return_value=self.WMI)): + self.assertTrue(win_dns_client.add_dns('10.1.1.0', 'Local Area Connection')) # 'dns_dhcp' function tests: 1 @@ -148,9 +152,10 @@ class WinDnsClientTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it get the type of DNS configuration (dhcp / static) ''' - with patch('salt.utils.winapi.Com', MagicMock()): - with patch.object(self.WMI, 'Win32_NetworkAdapter', - return_value=[Mockwmi()]): - with patch.object(self.WMI, 'Win32_NetworkAdapterConfiguration', - return_value=[Mockwmi()]): - self.assertTrue(win_dns_client.get_dns_config()) + with patch('salt.utils.winapi.Com', MagicMock()), \ + patch.object(self.WMI, 'Win32_NetworkAdapter', + return_value=[Mockwmi()]), \ + patch.object(self.WMI, 'Win32_NetworkAdapterConfiguration', + return_value=[Mockwmi()]), \ + patch.object(wmi, 'WMI', Mock(return_value=self.WMI)): + self.assertTrue(win_dns_client.get_dns_config()) diff --git a/tests/unit/modules/test_win_network.py b/tests/unit/modules/test_win_network.py index e849a0d8bf0..adb35572212 100644 --- a/tests/unit/modules/test_win_network.py +++ b/tests/unit/modules/test_win_network.py @@ -5,7 +5,6 @@ # Import Python Libs from __future__ import absolute_import -import types # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -22,6 +21,12 @@ from tests.support.mock import ( import salt.utils import salt.modules.win_network as win_network +try: + import wmi + HAS_WMI = True +except ImportError: + HAS_WMI = False + class Mockwmi(object): ''' @@ -65,10 +70,8 @@ class WinNetworkTestCase(TestCase, LoaderModuleMockMixin): ''' def setup_loader_modules(self): # wmi modules are platform specific... - wmi = types.ModuleType('wmi') self.WMI = Mock() self.addCleanup(delattr, self, 'WMI') - wmi.WMI = Mock(return_value=self.WMI) return {win_network: {'wmi': wmi}} # 'ping' function tests: 1 @@ -156,6 +159,7 @@ class WinNetworkTestCase(TestCase, LoaderModuleMockMixin): # 'interfaces_names' function tests: 1 + @skipIf(not HAS_WMI, "WMI only available on Windows") def test_interfaces_names(self): ''' Test if it return a list of all the interfaces names @@ -164,7 +168,8 @@ class WinNetworkTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.utils.winapi.Com', MagicMock()), \ patch.object(self.WMI, 'Win32_NetworkAdapter', return_value=[Mockwmi()]), \ - patch('salt.utils', Mockwinapi): + patch('salt.utils', Mockwinapi), \ + patch.object(wmi, 'WMI', Mock(return_value=self.WMI)): self.assertListEqual(win_network.interfaces_names(), ['Ethernet']) diff --git a/tests/unit/modules/test_win_status.py b/tests/unit/modules/test_win_status.py index df317323abd..b3322278cf9 100644 --- a/tests/unit/modules/test_win_status.py +++ b/tests/unit/modules/test_win_status.py @@ -3,7 +3,6 @@ # Import python libs from __future__ import absolute_import import sys -import types # Import Salt libs import salt.ext.six as six @@ -12,25 +11,16 @@ import salt.ext.six as six from tests.support.unit import skipIf, TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, Mock, patch, ANY -# wmi and pythoncom modules are platform specific... -wmi = types.ModuleType('wmi') -sys.modules['wmi'] = wmi - -pythoncom = types.ModuleType('pythoncom') -sys.modules['pythoncom'] = pythoncom - -if NO_MOCK is False: - WMI = Mock() - wmi.WMI = Mock(return_value=WMI) - pythoncom.CoInitialize = Mock() - pythoncom.CoUninitialize = Mock() +try: + import wmi +except ImportError: + pass # This is imported late so mock can do its job import salt.modules.win_status as status @skipIf(NO_MOCK, NO_MOCK_REASON) -@skipIf(sys.stdin.encoding != 'UTF-8', 'UTF-8 encoding required for this test is not supported') @skipIf(status.HAS_WMI is False, 'This test requires Windows') class TestProcsBase(TestCase): def __init__(self, *args, **kwargs): @@ -55,8 +45,10 @@ class TestProcsBase(TestCase): self.__processes.append(process) def call_procs(self): + WMI = Mock() WMI.win32_process = Mock(return_value=self.__processes) - self.result = status.procs() + with patch.object(wmi, 'WMI', Mock(return_value=WMI)): + self.result = status.procs() class TestProcsCount(TestProcsBase): @@ -101,6 +93,7 @@ class TestProcsAttributes(TestProcsBase): self.assertEqual(self.proc['user_domain'], self._expected_domain) +@skipIf(sys.stdin.encoding != 'UTF-8', 'UTF-8 encoding required for this test is not supported') class TestProcsUnicodeAttributes(TestProcsBase): def setUp(self): unicode_str = u'\xc1' From 20273e369706319962fd4fd6aad837fd68d8e7a0 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 10 Nov 2017 09:54:54 -0600 Subject: [PATCH 024/159] No need for setup_loader_modules since we're actually importing wmi --- tests/unit/modules/test_win_network.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/unit/modules/test_win_network.py b/tests/unit/modules/test_win_network.py index adb35572212..915cf39a5db 100644 --- a/tests/unit/modules/test_win_network.py +++ b/tests/unit/modules/test_win_network.py @@ -7,7 +7,6 @@ from __future__ import absolute_import # Import Salt Testing Libs -from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import TestCase, skipIf from tests.support.mock import ( MagicMock, @@ -64,15 +63,13 @@ class Mockwinapi(object): @skipIf(NO_MOCK, NO_MOCK_REASON) -class WinNetworkTestCase(TestCase, LoaderModuleMockMixin): +class WinNetworkTestCase(TestCase): ''' Test cases for salt.modules.win_network ''' - def setup_loader_modules(self): - # wmi modules are platform specific... + def setUp(self): self.WMI = Mock() self.addCleanup(delattr, self, 'WMI') - return {win_network: {'wmi': wmi}} # 'ping' function tests: 1 From 6c872e95e6962d76a7f0611ed904071c92f9afb5 Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 10 Nov 2017 11:07:32 -0700 Subject: [PATCH 025/159] Add back the setup_loader_modules function Use empty dict instead of defining wmi --- tests/unit/modules/test_win_network.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit/modules/test_win_network.py b/tests/unit/modules/test_win_network.py index 915cf39a5db..395a00cba63 100644 --- a/tests/unit/modules/test_win_network.py +++ b/tests/unit/modules/test_win_network.py @@ -7,6 +7,7 @@ from __future__ import absolute_import # Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import TestCase, skipIf from tests.support.mock import ( MagicMock, @@ -63,13 +64,14 @@ class Mockwinapi(object): @skipIf(NO_MOCK, NO_MOCK_REASON) -class WinNetworkTestCase(TestCase): +class WinNetworkTestCase(TestCase, LoaderModuleMockMixin): ''' Test cases for salt.modules.win_network ''' - def setUp(self): + def setup_loader_modules(self): self.WMI = Mock() self.addCleanup(delattr, self, 'WMI') + return {win_network: {}} # 'ping' function tests: 1 From c7cf5f6f7096bce40854a84fce4ddc68009812af Mon Sep 17 00:00:00 2001 From: Morgan Willcock Date: Sun, 12 Nov 2017 12:33:23 +0000 Subject: [PATCH 026/159] Format pywintypes.error --- salt/utils/win_dacl.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/salt/utils/win_dacl.py b/salt/utils/win_dacl.py index 923c1659756..6503dc277f6 100644 --- a/salt/utils/win_dacl.py +++ b/salt/utils/win_dacl.py @@ -1133,9 +1133,14 @@ def get_name(principal): try: return win32security.LookupAccountSid(None, sid_obj)[0] - except (pywintypes.error, TypeError): - raise CommandExecutionError( - 'Could not find User for {0}'.format(principal)) + except (pywintypes.error, TypeError) as exc: + if type(exc) == pywintypes.error: + win_error = win32api.FormatMessage(exc.winerror).rstrip('\n') + message = 'Error resolving {0} ({1})'.format(principal, win_error) + else: + message = 'Error resolving {0}'.format(principal) + + raise CommandExecutionError(message) def get_owner(obj_name): From f45378af04f753526cdc9c9bf37a6537f7db0367 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Mon, 13 Nov 2017 14:24:21 +0000 Subject: [PATCH 027/159] Lint: remove extra spaces --- salt/utils/napalm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/napalm.py b/salt/utils/napalm.py index 5597f991467..0608589a6ae 100644 --- a/salt/utils/napalm.py +++ b/salt/utils/napalm.py @@ -97,7 +97,7 @@ def virtual(opts, virtualname, filename): ''' Returns the __virtual__. ''' - if ( (HAS_NAPALM and NAPALM_MAJOR >= 2) or HAS_NAPALM_BASE ) and ( is_proxy(opts) or is_minion(opts) ): + if ((HAS_NAPALM and NAPALM_MAJOR >= 2) or HAS_NAPALM_BASE) and (is_proxy(opts) or is_minion(opts)): if HAS_NAPALM_BASE: log.info('You still seem to use napalm_base. Please consider upgrading to napalm >= 2.0.0') return virtualname From ffa4bddcadb3d7310db464e31eeaa8597c3750d3 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 13 Nov 2017 10:50:16 -0500 Subject: [PATCH 028/159] Increase sleep timeout for pillar refresh test --- tests/integration/modules/test_saltutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/modules/test_saltutil.py b/tests/integration/modules/test_saltutil.py index 17fa21acf78..ee90c74f3b9 100644 --- a/tests/integration/modules/test_saltutil.py +++ b/tests/integration/modules/test_saltutil.py @@ -186,7 +186,7 @@ class SaltUtilSyncPillarTest(ModuleCase): ''')) pillar_refresh = self.run_function('saltutil.refresh_pillar') - wait = self.run_function('test.sleep', [1]) + wait = self.run_function('test.sleep', [5]) post_pillar = self.run_function('pillar.raw') self.assertIn(pillar_key, post_pillar.get(pillar_key, 'didnotwork')) From 6e00e415d309fe4838bca8643f1f2cd2b5e0c2b2 Mon Sep 17 00:00:00 2001 From: mephi42 Date: Fri, 10 Nov 2017 11:48:14 +0100 Subject: [PATCH 029/159] nova: fix endpoint URL determination in _v3_setup() Some OpenStack setups return multiple entries for the same service. This change makes the code consider all matching entries instead of just the first one. --- salt/utils/openstack/nova.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/salt/utils/openstack/nova.py b/salt/utils/openstack/nova.py index 82deb5769cc..e1778e70bdf 100644 --- a/salt/utils/openstack/nova.py +++ b/salt/utils/openstack/nova.py @@ -191,6 +191,16 @@ def get_entry_multi(dict_, pairs, raise_error=True): return {} +def get_endpoint_url_v3(catalog, service_type, region_name): + for service_entry in catalog: + if service_entry['type'] == service_type: + for endpoint_entry in service_entry['endpoints']: + if (endpoint_entry['region'] == region_name and + endpoint_entry['interface'] == 'public'): + return endpoint_entry['url'] + return None + + def sanatize_novaclient(kwargs): variables = ( 'username', 'api_key', 'project_id', 'auth_url', 'insecure', @@ -353,21 +363,16 @@ class SaltNova(object): def _v3_setup(self, region_name): if region_name is not None: - servers_endpoints = get_entry(self.catalog, 'type', 'compute')['endpoints'] - self.kwargs['bypass_url'] = get_entry_multi( - servers_endpoints, - [('region', region_name), ('interface', 'public')] - )['url'] + self.client_kwargs['bypass_url'] = get_endpoint_url_v3(self.catalog, 'compute', region_name) + log.debug('Using Nova bypass_url: %s', self.client_kwargs['bypass_url']) self.compute_conn = client.Client(version=self.version, session=self.session, **self.client_kwargs) volume_endpoints = get_entry(self.catalog, 'type', 'volume', raise_error=False).get('endpoints', {}) if volume_endpoints: if region_name is not None: - self.kwargs['bypass_url'] = get_entry_multi( - volume_endpoints, - [('region', region_name), ('interface', 'public')] - )['url'] + self.client_kwargs['bypass_url'] = get_endpoint_url_v3(self.catalog, 'volume', region_name) + log.debug('Using Cinder bypass_url: %s', self.client_kwargs['bypass_url']) self.volume_conn = client.Client(version=self.version, session=self.session, **self.client_kwargs) if hasattr(self, 'extensions'): From 61f30a9d805cedc84466af2088d4b4051e18e9f7 Mon Sep 17 00:00:00 2001 From: Olivier Mauras Date: Tue, 3 Oct 2017 20:51:03 +0200 Subject: [PATCH 030/159] Correct when dict is NoneType because of the value doesn't exist Example: ``` yaml pillars: default: commands: {% if grains['os'] == 'RedHat' %} pkg_grains: /usr/bin/rpm_grains {% endif %} ``` In this case `pkg_grains` wouldn't be found on anything else that RedHat systems and would be NoneType which would return an error and break the module. --- salt/utils/saltclass.py | 5 +++-- .../saltclass/examples/classes/default/init.yml | 2 ++ .../files/saltclass/examples/nodes/fake_id.yml | 2 +- tests/unit/pillar/test_saltclass.py | 13 ++++++++++--- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/salt/utils/saltclass.py b/salt/utils/saltclass.py index 3df204d5dc1..ab8dd0adb1d 100644 --- a/salt/utils/saltclass.py +++ b/salt/utils/saltclass.py @@ -104,11 +104,12 @@ def dict_search_and_replace(d, old, new, expanded): def find_value_to_expand(x, v): a = x for i in v[2:-1].split(':'): + if a is None: + return v if i in a: a = a.get(i) else: - a = v - return a + return v return a diff --git a/tests/integration/files/saltclass/examples/classes/default/init.yml b/tests/integration/files/saltclass/examples/classes/default/init.yml index 20a5e450883..7798ced203d 100644 --- a/tests/integration/files/saltclass/examples/classes/default/init.yml +++ b/tests/integration/files/saltclass/examples/classes/default/init.yml @@ -9,9 +9,11 @@ pillars: default: network: dns: +{% if __grains__['os'] == 'should_never_match' %} srv1: 192.168.0.1 srv2: 192.168.0.2 domain: example.com +{% endif %} ntp: srv1: 192.168.10.10 srv2: 192.168.10.20 diff --git a/tests/integration/files/saltclass/examples/nodes/fake_id.yml b/tests/integration/files/saltclass/examples/nodes/fake_id.yml index a87137e6fbe..e2172c57d4c 100644 --- a/tests/integration/files/saltclass/examples/nodes/fake_id.yml +++ b/tests/integration/files/saltclass/examples/nodes/fake_id.yml @@ -1,6 +1,6 @@ environment: base classes: -{% for class in ['default'] %} +{% for class in ['default', 'roles.app'] %} - {{ class }} {% endfor %} diff --git a/tests/unit/pillar/test_saltclass.py b/tests/unit/pillar/test_saltclass.py index 30b63f8c548..9bbb806e071 100644 --- a/tests/unit/pillar/test_saltclass.py +++ b/tests/unit/pillar/test_saltclass.py @@ -34,10 +34,17 @@ class SaltclassPillarTestCase(TestCase, LoaderModuleMockMixin): }} def _runner(self, expected_ret): - full_ret = saltclass.ext_pillar(fake_minion_id, fake_pillar, fake_args) - parsed_ret = full_ret['__saltclass__']['classes'] + full_ret = {} + parsed_ret = [] + try: + full_ret = saltclass.ext_pillar(fake_minion_id, fake_pillar, fake_args) + parsed_ret = full_ret['__saltclass__']['classes'] + # Fail the test if we hit our NoneType error + except TypeError as err: + self.fail(err) + # Else give the parsed content result self.assertListEqual(parsed_ret, expected_ret) def test_succeeds(self): - ret = ['default.users', 'default.motd', 'default'] + ret = ['default.users', 'default.motd', 'default', 'roles.app'] self._runner(ret) From 47114fdb3097a6d55ee3ced9494286b5885832c8 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 13 Nov 2017 13:05:23 -0700 Subject: [PATCH 031/159] Pass root_dirs to the win_verify_env function Remove while loop that was hanging when `salt` wasn't in the path Add salt.util.path.safe_path function to check for unsafe paths Pass root_dir to all calls to `verify_env` --- salt/cli/daemons.py | 4 +++ salt/cli/spm.py | 5 ++- salt/cloud/cli.py | 3 +- salt/modules/win_pkg.py | 30 +++-------------- salt/utils/path.py | 57 +++++++++++++++++++++++++++++++++ salt/utils/verify.py | 19 +++++++---- tests/integration/__init__.py | 4 ++- tests/support/mixins.py | 4 ++- tests/unit/utils/test_verify.py | 2 +- 9 files changed, 90 insertions(+), 38 deletions(-) diff --git a/salt/cli/daemons.py b/salt/cli/daemons.py index a510ad60c1a..d427479d894 100644 --- a/salt/cli/daemons.py +++ b/salt/cli/daemons.py @@ -160,6 +160,7 @@ class Master(parsers.MasterOptionParser, DaemonsMixin): # pylint: disable=no-in self.config['user'], permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], + root_dir=self.config['root_dir'], ) # Clear out syndics from cachedir for syndic_file in os.listdir(self.config['syndic_dir']): @@ -280,6 +281,7 @@ class Minion(parsers.MinionOptionParser, DaemonsMixin): # pylint: disable=no-in self.config['user'], permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], + root_dir=self.config['root_dir'], ) except OSError as error: self.environment_failure(error) @@ -467,6 +469,7 @@ class ProxyMinion(parsers.ProxyMinionOptionParser, DaemonsMixin): # pylint: dis self.config['user'], permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], + root_dir=self.config['root_dir'], ) except OSError as error: self.environment_failure(error) @@ -575,6 +578,7 @@ class Syndic(parsers.SyndicOptionParser, DaemonsMixin): # pylint: disable=no-in self.config['user'], permissive=self.config['permissive_pki_access'], pki_dir=self.config['pki_dir'], + root_dir=self.config['root_dir'], ) except OSError as error: self.environment_failure(error) diff --git a/salt/cli/spm.py b/salt/cli/spm.py index 303e5ce65f4..3cecc769053 100644 --- a/salt/cli/spm.py +++ b/salt/cli/spm.py @@ -32,7 +32,10 @@ class SPM(parsers.SPMParser): v_dirs = [ self.config['cachedir'], ] - verify_env(v_dirs, self.config['user'],) + verify_env(v_dirs, + self.config['user'], + root_dir=self.config['root_dir'], + ) verify_log(self.config) client = salt.spm.SPMClient(ui, self.config) client.run(self.args) diff --git a/salt/cloud/cli.py b/salt/cloud/cli.py index fb8cdd25e15..04c4c349a56 100644 --- a/salt/cloud/cli.py +++ b/salt/cloud/cli.py @@ -66,7 +66,8 @@ class SaltCloud(parsers.SaltCloudParser): if self.config['verify_env']: verify_env( [os.path.dirname(self.config['conf_file'])], - salt_master_user + salt_master_user, + root_dir=self.config['root_dir'], ) logfile = self.config['log_file'] if logfile is not None and not logfile.startswith('tcp://') \ diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 2c7a2b5e010..c8f699a7d6c 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -52,6 +52,7 @@ from salt.exceptions import (CommandExecutionError, SaltRenderError) import salt.utils import salt.utils.pkg +import salt.utils.path import salt.syspaths import salt.payload from salt.exceptions import MinionError @@ -641,33 +642,10 @@ def _get_repo_details(saltenv): # Do some safety checks on the repo_path as its contents can be removed, # this includes check for bad coding system_root = os.environ.get('SystemRoot', r'C:\Windows') - deny_paths = ( - r'[a-z]\:\\$', # C:\, D:\, etc - r'\\$', # \ - re.escape(system_root) # C:\Windows - ) + if not salt.utils.path.safe_path( + path=local_dest, + allow_path='\\'.join([system_root, 'TEMP'])): - # Since the above checks anything in C:\Windows, there are some - # directories we may want to make exceptions for - allow_paths = ( - re.escape('\\'.join([system_root, 'TEMP'])), # C:\Windows\TEMP - ) - - # Check the local_dest to make sure it's not one of the bad paths - good_path = True - for d_path in deny_paths: - if re.match(d_path, local_dest, flags=re.IGNORECASE) is not None: - # Found deny path - good_path = False - - # If local_dest is one of the bad paths, check for exceptions - if not good_path: - for a_path in allow_paths: - if re.match(a_path, local_dest, flags=re.IGNORECASE) is not None: - # Found exception - good_path = True - - if not good_path: raise CommandExecutionError( 'Attempting to delete files from a possibly unsafe location: ' '{0}'.format(local_dest) diff --git a/salt/utils/path.py b/salt/utils/path.py index 90f7a7f53a4..faf43bfd2de 100644 --- a/salt/utils/path.py +++ b/salt/utils/path.py @@ -174,3 +174,60 @@ def _get_reparse_data(path): win32file.CloseHandle(fileHandle) return reparseData + + +def safe_path(path, allow_path=None): + r''' + .. versionadded:: 2017.7.3 + + Checks that the path is safe for modification by Salt. For example, you + wouldn't want to have salt delete the contents of ``C:\Windows``. The + following directories are considered unsafe: + + - C:\, D:\, E:\, etc. + - \ + - C:\Windows + + Args: + + path (str): The path to check + + allow_paths (str, list): A directory or list of directories inside of + path that may be safe. For example: ``C:\Windows\TEMP`` + + Returns: + bool: True if safe, otherwise False + ''' + # Create regex definitions for directories that may be unsafe to modify + system_root = os.environ.get('SystemRoot', 'C:\\Windows') + deny_paths = ( + r'[a-z]\:\\$', # C:\, D:\, etc + r'\\$', # \ + re.escape(system_root) # C:\Windows + ) + + # Make allow_path a list + if allow_path and not isinstance(allow_path, list): + allow_path = [allow_path] + + # Create regex definition for directories we may want to make exceptions for + allow_paths = list() + if allow_path: + for item in allow_path: + allow_paths.append(re.escape(item)) + + # Check the path to make sure it's not one of the bad paths + good_path = True + for d_path in deny_paths: + if re.match(d_path, path, flags=re.IGNORECASE) is not None: + # Found deny path + good_path = False + + # If local_dest is one of the bad paths, check for exceptions + if not good_path: + for a_path in allow_paths: + if re.match(a_path, path, flags=re.IGNORECASE) is not None: + # Found exception + good_path = True + + return good_path diff --git a/salt/utils/verify.py b/salt/utils/verify.py index 24b2a1d962e..821b5a4d605 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -31,6 +31,8 @@ import salt.utils log = logging.getLogger(__name__) +ROOT_DIR = 'c:\\salt' if salt.utils.is_windows() else '/' + def zmq_version(): ''' @@ -192,13 +194,13 @@ def verify_files(files, user): return True -def verify_env(dirs, user, permissive=False, pki_dir='', skip_extra=False): +def verify_env(dirs, user, permissive=False, pki_dir='', skip_extra=False, root_dir=ROOT_DIR): ''' Verify that the named directories are in place and that the environment can shake the salt ''' if salt.utils.is_windows(): - return win_verify_env(dirs, permissive, pki_dir, skip_extra) + return win_verify_env(root_dir, dirs, permissive, pki_dir, skip_extra) import pwd # after confirming not running Windows try: pwnam = pwd.getpwnam(user) @@ -523,18 +525,21 @@ def verify_log(opts): log.warning('Insecure logging configuration detected! Sensitive data may be logged.') -def win_verify_env(dirs, permissive=False, pki_dir='', skip_extra=False): +def win_verify_env(path, dirs, permissive=False, pki_dir='', skip_extra=False): ''' Verify that the named directories are in place and that the environment can shake the salt ''' import salt.utils.win_functions import salt.utils.win_dacl + import salt.utils.path - # Get the root path directory where salt is installed - path = dirs[0] - while os.path.basename(path) not in ['salt', 'salt-tests-tmpdir']: - path, base = os.path.split(path) + # Make sure the file_roots is not set to something unsafe since permissions + # on that directory are reset + if not salt.utils.path.safe_path(path=path): + raise CommandExecutionError( + '`file_roots` set to a possibly unsafe location: {0}'.format(path) + ) # Create the root path directory if missing if not os.path.isdir(path): diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 378b2a12ad0..17ecb2d1728 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -983,7 +983,9 @@ class TestDaemon(object): RUNTIME_VARS.TMP_PRODENV_STATE_TREE, TMP, ], - RUNTIME_VARS.RUNNING_TESTS_USER) + RUNTIME_VARS.RUNNING_TESTS_USER, + root_dir=master_opts['root_dir'], + ) cls.master_opts = master_opts cls.minion_opts = minion_opts diff --git a/tests/support/mixins.py b/tests/support/mixins.py index 38601d81ff1..00aad6be195 100644 --- a/tests/support/mixins.py +++ b/tests/support/mixins.py @@ -107,7 +107,9 @@ class AdaptedConfigurationTestCaseMixin(object): rdict['sock_dir'], conf_dir ], - RUNTIME_VARS.RUNNING_TESTS_USER) + RUNTIME_VARS.RUNNING_TESTS_USER, + root_dir=rdict['root_dir'], + ) rdict['config_dir'] = conf_dir rdict['conf_file'] = os.path.join(conf_dir, config_for) diff --git a/tests/unit/utils/test_verify.py b/tests/unit/utils/test_verify.py index c1bd942adf7..d9a67535a26 100644 --- a/tests/unit/utils/test_verify.py +++ b/tests/unit/utils/test_verify.py @@ -111,7 +111,7 @@ class TestVerify(TestCase): def test_verify_env(self): root_dir = tempfile.mkdtemp(dir=TMP) var_dir = os.path.join(root_dir, 'var', 'log', 'salt') - verify_env([var_dir], getpass.getuser()) + verify_env([var_dir], getpass.getuser(), root_dir=root_dir) self.assertTrue(os.path.exists(var_dir)) dir_stat = os.stat(var_dir) self.assertEqual(dir_stat.st_uid, os.getuid()) From 3b93ea058bea1adaf8c6f8665948ace3bd4e5d08 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Mon, 13 Nov 2017 15:53:09 -0700 Subject: [PATCH 032/159] ubuntu 14 and centos 6 should not have py3 tests --- .kitchen.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.kitchen.yml b/.kitchen.yml index ef1c63579f1..9273e30f326 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -164,6 +164,9 @@ suites: clone_repo: false salttesting_namespec: salttesting==2017.6.1 - name: py3 + excludes: + - centos-6 + - ubuntu-14.04 provisioner: pillars: top.sls: From c2210aaf7ce6a378546f5306c48c56dd03f5b491 Mon Sep 17 00:00:00 2001 From: Tom Williams Date: Tue, 14 Nov 2017 02:02:14 -0500 Subject: [PATCH 033/159] INFRA-5978 - fix for https://github.com/saltstack/salt/issues/44290 --- salt/states/boto_vpc.py | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/salt/states/boto_vpc.py b/salt/states/boto_vpc.py index b88f53541c3..56a2597cec5 100644 --- a/salt/states/boto_vpc.py +++ b/salt/states/boto_vpc.py @@ -1496,13 +1496,8 @@ def accept_vpc_peering_connection(name=None, conn_id=None, conn_name=None, ''' log.debug('Called state to accept VPC peering connection') pending = __salt__['boto_vpc.is_peering_connection_pending']( - conn_id=conn_id, - conn_name=conn_name, - region=region, - key=key, - keyid=keyid, - profile=profile - ) + conn_id=conn_id, conn_name=conn_name, region=region, key=key, + keyid=keyid, profile=profile) ret = { 'name': name, @@ -1511,32 +1506,27 @@ def accept_vpc_peering_connection(name=None, conn_id=None, conn_name=None, 'comment': 'Boto VPC peering state' } - if not pending['exists']: + if not pending: ret['result'] = True - ret['changes'].update({ - 'old': 'No pending VPC peering connection found. ' - 'Nothing to be done.' - }) + ret['changes'].update({'old': + 'No pending VPC peering connection found. Nothing to be done.' }) return ret if __opts__['test']: - ret['changes'].update({'old': 'Pending VPC peering connection found ' - 'and can be accepted'}) + ret['changes'].update({'old': + 'Pending VPC peering connection found and can be accepted'}) return ret - log.debug('Calling module to accept this VPC peering connection') - result = __salt__['boto_vpc.accept_vpc_peering_connection']( - conn_id=conn_id, name=conn_name, region=region, key=key, + fun = 'boto_vpc.accept_vpc_peering_connection' + log.debug('Calling `{0}()` to accept this VPC peering connection'.format(fun)) + result = __salt__[fun](conn_id=conn_id, name=conn_name, region=region, key=key, keyid=keyid, profile=profile) if 'error' in result: - ret['comment'] = "Failed to request VPC peering: {0}".format(result['error']) + ret['comment'] = "Failed to accept VPC peering: {0}".format(result['error']) ret['result'] = False return ret - ret['changes'].update({ - 'old': '', - 'new': result['msg'] - }) + ret['changes'].update({'old': '', 'new': result['msg']}) return ret From 9bc70fd31bcf6f7972337d39832c8c1092dca38f Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 14 Nov 2017 11:01:17 -0700 Subject: [PATCH 034/159] back up to 2017.7.1 for kitchen tests Until https://github.com/saltstack/salt/pull/44016 is included centos tests need to use 2017.7.1, otherwise it is unable to download the busybox binary. --- .kitchen.yml | 2 +- Gemfile | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index 9273e30f326..d39eb054bf2 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -1,6 +1,6 @@ --- <% vagrant = system('which vagrant 2>/dev/null >/dev/null') %> -<% version = '2017.7.2' %> +<% version = '2017.7.1' %> <% platformsfile = ENV['SALT_KITCHEN_PLATFORMS'] || '.kitchen/platforms.yml' %> <% driverfile = ENV['SALT_KITCHEN_DRIVER'] || '.kitchen/driver.yml' %> diff --git a/Gemfile b/Gemfile index a29160e2f4e..18f82b9ffca 100644 --- a/Gemfile +++ b/Gemfile @@ -1,9 +1,9 @@ # This file is only used for running the test suite with kitchen-salt. -source "https://rubygems.org" +source 'https://rubygems.org' -gem "test-kitchen" -gem "kitchen-salt", :git => 'https://github.com/saltstack/kitchen-salt.git' +gem 'test-kitchen' +gem 'kitchen-salt', :git => 'https://github.com/saltstack/kitchen-salt.git' gem 'git' group :docker do From 3e04d2d44c6c6f19b94cb8ef7115b90873127b24 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Tue, 14 Nov 2017 11:39:44 -0700 Subject: [PATCH 035/159] use kitchen-sync for copying files --- .kitchen.yml | 2 ++ Gemfile | 1 + 2 files changed, 3 insertions(+) diff --git a/.kitchen.yml b/.kitchen.yml index d39eb054bf2..a3acdc3abfa 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -19,6 +19,8 @@ driver: disable_upstart: false provision_command: - echo 'L /run/docker.sock - - - - /docker.sock' > /etc/tmpfiles.d/docker.conf +transport: + name: sftp <% end %> sudo: false diff --git a/Gemfile b/Gemfile index 18f82b9ffca..b63a01dbc7b 100644 --- a/Gemfile +++ b/Gemfile @@ -4,6 +4,7 @@ source 'https://rubygems.org' gem 'test-kitchen' gem 'kitchen-salt', :git => 'https://github.com/saltstack/kitchen-salt.git' +gem 'kitchen-sync' gem 'git' group :docker do From 958e1aeb8d55fe48249b4812707b4d73d172eca9 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 14 Nov 2017 13:43:39 -0600 Subject: [PATCH 036/159] Fix test to reflect changes in YAML dumper PR #42064 modified the YAML dumper to display unicode text as a unicode literal rather than with the !!python/unicode extension prefix. This updates the test to reflect the change in Salt's behavior. --- tests/unit/templates/test_jinja.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/templates/test_jinja.py b/tests/unit/templates/test_jinja.py index 4ad4b618f81..a5be05056d4 100644 --- a/tests/unit/templates/test_jinja.py +++ b/tests/unit/templates/test_jinja.py @@ -561,7 +561,6 @@ class TestCustomExtensions(TestCase): # type of the rendered variable (should be unicode, which is the same as # six.text_type). This should cover all use cases but also allow the test # to pass on CentOS 6 running Python 2.7. - self.assertIn('!!python/unicode', rendered) self.assertIn('str value', rendered) self.assertIsInstance(rendered, six.text_type) From 021692b6c91c0c88e68bebf920f99ad3caf5ec90 Mon Sep 17 00:00:00 2001 From: Tom Williams Date: Tue, 14 Nov 2017 16:29:54 -0500 Subject: [PATCH 037/159] INFRA-5978 - pylint / whitespace fix --- salt/states/boto_vpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/boto_vpc.py b/salt/states/boto_vpc.py index 56a2597cec5..57cd0bd4d6d 100644 --- a/salt/states/boto_vpc.py +++ b/salt/states/boto_vpc.py @@ -1509,7 +1509,7 @@ def accept_vpc_peering_connection(name=None, conn_id=None, conn_name=None, if not pending: ret['result'] = True ret['changes'].update({'old': - 'No pending VPC peering connection found. Nothing to be done.' }) + 'No pending VPC peering connection found. Nothing to be done.'}) return ret if __opts__['test']: From 2d6176b0bca6ec94a2884329aaad5f1b3242b6d5 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 15 Nov 2017 13:35:56 +0000 Subject: [PATCH 038/159] Fx2 proxy minion: clean return, like all the other modules --- salt/proxy/fx2.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/proxy/fx2.py b/salt/proxy/fx2.py index e90c345a45f..c4e262b355c 100644 --- a/salt/proxy/fx2.py +++ b/salt/proxy/fx2.py @@ -196,9 +196,7 @@ def __virtual__(): Only return if all the modules are available ''' if not salt.utils.which('racadm'): - log.critical('fx2 proxy minion needs "racadm" to be installed.') - return False - + return False, 'fx2 proxy minion needs "racadm" to be installed.' return True From 6689bd3b2df41a92b256b2351611812fa7cf339c Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 15 Nov 2017 13:34:49 +0000 Subject: [PATCH 039/159] Dont use dangerous os.tmpnam --- salt/proxy/dummy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/proxy/dummy.py b/salt/proxy/dummy.py index eeb0fb0b3e4..a7f74f0799d 100644 --- a/salt/proxy/dummy.py +++ b/salt/proxy/dummy.py @@ -6,8 +6,9 @@ from __future__ import absolute_import # Import python libs import os -import logging import pickle +import logging +import tempfile # This must be present or the Salt loader won't load this module __proxyenabled__ = ['dummy'] @@ -19,7 +20,7 @@ DETAILS = {} DETAILS['services'] = {'apache': 'running', 'ntp': 'running', 'samba': 'stopped'} DETAILS['packages'] = {'coreutils': '1.0', 'apache': '2.4', 'tinc': '1.4', 'redbull': '999.99'} -FILENAME = os.tmpnam() +FILENAME = tempfile.mkstemp()[1] # Want logging! log = logging.getLogger(__file__) From ce1882943d72b7b15948f3e42d64e1c06fdab12e Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Wed, 15 Nov 2017 15:14:58 +0000 Subject: [PATCH 040/159] Use salt.utils.files.mkstemp() instead --- salt/proxy/dummy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/proxy/dummy.py b/salt/proxy/dummy.py index a7f74f0799d..70933e86e5a 100644 --- a/salt/proxy/dummy.py +++ b/salt/proxy/dummy.py @@ -8,7 +8,9 @@ from __future__ import absolute_import import os import pickle import logging -import tempfile + +# Import Salt modules +import salt.utils.files # This must be present or the Salt loader won't load this module __proxyenabled__ = ['dummy'] @@ -20,7 +22,7 @@ DETAILS = {} DETAILS['services'] = {'apache': 'running', 'ntp': 'running', 'samba': 'stopped'} DETAILS['packages'] = {'coreutils': '1.0', 'apache': '2.4', 'tinc': '1.4', 'redbull': '999.99'} -FILENAME = tempfile.mkstemp()[1] +FILENAME = salt.utils.files.mkstemp() # Want logging! log = logging.getLogger(__file__) From 913eedc699274c0d8cbfbb25794a250641bfa563 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 14 Nov 2017 11:08:48 -0500 Subject: [PATCH 041/159] Add multiple salt-ssh state integration tests --- tests/integration/ssh/test_state.py | 91 ++++++++++++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/tests/integration/ssh/test_state.py b/tests/integration/ssh/test_state.py index ba78a1048ef..26343b4efed 100644 --- a/tests/integration/ssh/test_state.py +++ b/tests/integration/ssh/test_state.py @@ -7,8 +7,13 @@ import shutil # Import Salt Testing Libs from tests.support.case import SSHCase +from tests.support.paths import TMP + +# Import Salt Libs +from salt.ext import six SSH_SLS = 'ssh_state_tests' +SSH_SLS_FILE = '/tmp/test' class SSHStateTest(SSHCase): @@ -37,6 +42,87 @@ class SSHStateTest(SSHCase): check_file = self.run_function('file.file_exists', ['/tmp/test']) self.assertTrue(check_file) + def test_state_show_sls(self): + ''' + test state.show_sls with salt-ssh + ''' + ret = self.run_function('state.show_sls', [SSH_SLS]) + self._check_dict_ret(ret=ret, val='__sls__', exp_ret=SSH_SLS) + + check_file = self.run_function('file.file_exists', [SSH_SLS_FILE], wipe=False) + self.assertFalse(check_file) + + def test_state_show_top(self): + ''' + test state.show_top with salt-ssh + ''' + ret = self.run_function('state.show_top') + self.assertEqual(ret, {u'base': [u'master_tops_test', u'core']}) + + def test_state_single(self): + ''' + state.single with salt-ssh + ''' + ret_out = {'name': 'itworked', + 'result': True, + 'comment': 'Success!'} + + single = self.run_function('state.single', + ['test.succeed_with_changes name=itworked']) + + for key, value in six.iteritems(single): + self.assertEqual(value['name'], ret_out['name']) + self.assertEqual(value['result'], ret_out['result']) + self.assertEqual(value['comment'], ret_out['comment']) + + def test_show_highstate(self): + ''' + state.show_highstate with salt-ssh + ''' + high = self.run_function('state.show_highstate') + destpath = os.path.join(TMP, 'testfile') + self.assertTrue(isinstance(high, dict)) + self.assertTrue(destpath in high) + self.assertEqual(high[destpath]['__env__'], 'base') + + def test_state_high(self): + ''' + state.high with salt-ssh + ''' + ret_out = {'name': 'itworked', + 'result': True, + 'comment': 'Success!'} + + high = self.run_function('state.high', ['"{"itworked": {"test": ["succeed_with_changes"]}}"']) + + for key, value in six.iteritems(high): + self.assertEqual(value['name'], ret_out['name']) + self.assertEqual(value['result'], ret_out['result']) + self.assertEqual(value['comment'], ret_out['comment']) + + def test_show_lowstate(self): + ''' + state.show_lowstate with salt-ssh + ''' + low = self.run_function('state.show_lowstate') + self.assertTrue(isinstance(low, list)) + self.assertTrue(isinstance(low[0], dict)) + + def test_state_low(self): + ''' + state.low with salt-ssh + ''' + ret_out = {'name': 'itworked', + 'result': True, + 'comment': 'Success!'} + + low = self.run_function('state.low', ['"{"state": "test", "fun": "succeed_with_changes", "name": "itworked"}"']) + + for key, value in six.iteritems(low): + self.assertEqual(value['name'], ret_out['name']) + self.assertEqual(value['result'], ret_out['result']) + self.assertEqual(value['comment'], ret_out['comment']) + def test_state_request_check_clear(self): ''' test state.request system with salt-ssh @@ -60,7 +146,7 @@ class SSHStateTest(SSHCase): run = self.run_function('state.run_request', wipe=False) - check_file = self.run_function('file.file_exists', ['/tmp/test'], wipe=False) + check_file = self.run_function('file.file_exists', [SSH_SLS_FILE], wipe=False) self.assertTrue(check_file) def tearDown(self): @@ -70,3 +156,6 @@ class SSHStateTest(SSHCase): salt_dir = self.run_function('config.get', ['thin_dir'], wipe=False) if os.path.exists(salt_dir): shutil.rmtree(salt_dir) + + if os.path.exists(SSH_SLS_FILE): + os.remove(SSH_SLS_FILE) From b98df6de247d52c875ecf59568d2a93c8bd7abaf Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Thu, 16 Nov 2017 11:10:21 -0500 Subject: [PATCH 042/159] Add known_hosts_file to salt-ssh opts_pkg in wfuncs --- salt/client/ssh/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index a25e8838b54..5637f14ab0e 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -903,6 +903,8 @@ class Single(object): ret = json.dumps({'local': opts_pkg}) return ret, retcode + if 'known_hosts_file' in self.opts: + opts_pkg['known_hosts_file'] = self.opts['known_hosts_file'] opts_pkg['file_roots'] = self.opts['file_roots'] opts_pkg['pillar_roots'] = self.opts['pillar_roots'] opts_pkg['ext_pillar'] = self.opts['ext_pillar'] From d341e12d048af56f6b90f319481ff174999b1c24 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 16 Nov 2017 08:44:31 -0800 Subject: [PATCH 043/159] Adding a bit of code to allow using requisites without a module name. Account for using the name without the state module in the requisite. Adding the ability to specify options to the listen state parameter without specifying the state mdoule. Adding state files used by new integration tests. --- salt/state.py | 42 ++++++- tests/integration/modules/test_state.py | 141 ++++++++++++++++++++++++ 2 files changed, 179 insertions(+), 4 deletions(-) diff --git a/salt/state.py b/salt/state.py index 649a83f1b88..54c3bb9b91f 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1601,7 +1601,24 @@ class State(object): for ind in items: if not isinstance(ind, dict): # Malformed req_in - continue + if ind in high: + _ind_high = [x for x + in high[ind] + if not x.startswith('__')] + ind = {_ind_high[0]: ind} + else: + found = False + for _id in iter(high): + for state in [state for state + in iter(high[_id]) + if not state.startswith('__')]: + for j in iter(high[_id][state]): + if isinstance(j, dict) and 'name' in j: + if j['name'] == ind: + ind = {state: _id} + found = True + if not found: + continue if len(ind) < 1: continue pstate = next(iter(ind)) @@ -2512,7 +2529,7 @@ class State(object): self.event(running[tag], len(chunks), fire_event=low.get(u'fire_event')) return running - def call_listen(self, chunks, running): + def call_listen(self, chunks, running, high): ''' Find all of the listen routines and call the associated mod_watch runs ''' @@ -2533,7 +2550,24 @@ class State(object): for key, val in six.iteritems(l_dict): for listen_to in val: if not isinstance(listen_to, dict): - continue + if listen_to in high: + _listen_high = [x for x + in high[listen_to] + if not x.startswith('__')] + listen_to = {_listen_high[0]: listen_to} + else: + found = False + for _id in iter(high): + for state in [state for state + in iter(high[_id]) + if not state.startswith('__')]: + for j in iter(high[_id][state]): + if isinstance(j, dict) and 'name' in j: + if j['name'] == listen_to: + listen_to = {state: _id} + found = True + if not found: + continue for lkey, lval in six.iteritems(listen_to): if (lkey, lval) not in crefs: rerror = {_l_tag(lkey, lval): @@ -2599,7 +2633,7 @@ class State(object): if errors: return errors ret = self.call_chunks(chunks) - ret = self.call_listen(chunks, ret) + ret = self.call_listen(chunks, ret, high) def _cleanup_accumulator_data(): accum_data_path = os.path.join( diff --git a/tests/integration/modules/test_state.py b/tests/integration/modules/test_state.py index 4631454b1a3..0f5469b1d94 100644 --- a/tests/integration/modules/test_state.py +++ b/tests/integration/modules/test_state.py @@ -730,6 +730,65 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): #ret = self.run_function('state.sls', mods='requisites.fullsls_prereq') #self.assertEqual(['sls command can only be used with require requisite'], ret) + def test_requisites_require_no_state_module(self): + ''' + Call sls file containing several require_in and require. + + Ensure that some of them are failing and that the order is right. + ''' + expected_result = { + 'cmd_|-A_|-echo A fifth_|-run': { + '__run_num__': 4, + 'comment': 'Command "echo A fifth" run', + 'result': True, + 'changes': True, + }, + 'cmd_|-B_|-echo B second_|-run': { + '__run_num__': 1, + 'comment': 'Command "echo B second" run', + 'result': True, + 'changes': True, + }, + 'cmd_|-C_|-echo C third_|-run': { + '__run_num__': 2, + 'comment': 'Command "echo C third" run', + 'result': True, + 'changes': True, + }, + 'cmd_|-D_|-echo D first_|-run': { + '__run_num__': 0, + 'comment': 'Command "echo D first" run', + 'result': True, + 'changes': True, + }, + 'cmd_|-E_|-echo E fourth_|-run': { + '__run_num__': 3, + 'comment': 'Command "echo E fourth" run', + 'result': True, + 'changes': True, + }, + 'cmd_|-G_|-echo G_|-run': { + '__run_num__': 5, + 'comment': 'The following requisites were not found:\n' + + ' require:\n' + + ' id: Z\n', + 'result': False, + 'changes': False, + }, + 'cmd_|-H_|-echo H_|-run': { + '__run_num__': 6, + 'comment': 'The following requisites were not found:\n' + + ' require:\n' + + ' id: Z\n', + 'result': False, + 'changes': False, + } + } + ret = self.run_function('state.sls', mods='requisites.require_no_state_module') + result = self.normalize_ret(ret) + self.assertReturnNonEmptySaltType(ret) + self.assertEqual(expected_result, result) + def test_requisites_prereq_simple_ordering_and_errors(self): ''' Call sls file containing several prereq_in and prereq. @@ -767,6 +826,30 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): 'result': False, 'changes': False} } + expected_result_simple_no_state_module = { + 'cmd_|-A_|-echo A third_|-run': { + '__run_num__': 2, + 'comment': 'Command "echo A third" run', + 'result': True, + 'changes': True}, + 'cmd_|-B_|-echo B first_|-run': { + '__run_num__': 0, + 'comment': 'Command "echo B first" run', + 'result': True, + 'changes': True}, + 'cmd_|-C_|-echo C second_|-run': { + '__run_num__': 1, + 'comment': 'Command "echo C second" run', + 'result': True, + 'changes': True}, + 'cmd_|-I_|-echo I_|-run': { + '__run_num__': 3, + 'comment': 'The following requisites were not found:\n' + + ' prereq:\n' + + ' id: Z\n', + 'result': False, + 'changes': False} + } expected_result_simple2 = { 'cmd_|-A_|-echo A_|-run': { '__run_num__': 1, @@ -897,6 +980,11 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): # ret, # ['A recursive requisite was found, SLS "requisites.prereq_recursion_error" ID "B" ID "A"'] #) + + ret = self.run_function('state.sls', mods='requisites.prereq_simple_no_state_module') + result = self.normalize_ret(ret) + self.assertEqual(expected_result_simple_no_state_module, result) + def test_infinite_recursion_sls_prereq(self): ret = self.run_function('state.sls', mods='requisites.prereq_sls_infinite_recursion') self.assertSaltTrueReturn(ret) @@ -933,6 +1021,16 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): # + ' ID "A" ID "A"' #]) + def test_requisites_use_no_state_module(self): + ''' + Call sls file containing several use_in and use. + + ''' + ret = self.run_function('state.sls', mods='requisites.use_no_state_module') + self.assertReturnNonEmptySaltType(ret) + for item, descr in six.iteritems(ret): + self.assertEqual(descr['comment'], 'onlyif execution failed') + def test_get_file_from_env_in_top_match(self): tgt = os.path.join(TMP, 'prod-cheese-file') try: @@ -1010,6 +1108,16 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): expected_result = 'State was not run because none of the onchanges reqs changed' self.assertIn(expected_result, test_data) + def test_onchanges_requisite_no_state_module(self): + ''' + Tests a simple state using the onchanges requisite without state modules + ''' + # Only run the state once and keep the return data + state_run = self.run_function('state.sls', mods='requisites.onchanges_simple_no_state_module') + test_data = state_run['cmd_|-test_changing_state_|-echo "Success!"_|-run']['comment'] + expected_result = 'Command "echo "Success!"" run' + self.assertIn(expected_result, test_data) + # onfail tests def test_onfail_requisite(self): @@ -1063,6 +1171,24 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): expected_result = 'State was not run because onfail req did not change' self.assertIn(expected_result, test_data) + def test_onfail_requisite_no_state_module(self): + ''' + Tests a simple state using the onfail requisite + ''' + + # Only run the state once and keep the return data + state_run = self.run_function('state.sls', mods='requisites.onfail_simple_no_state_module') + + # First, test the result of the state run when a failure is expected to happen + test_data = state_run['cmd_|-test_failing_state_|-echo "Success!"_|-run']['comment'] + expected_result = 'Command "echo "Success!"" run' + self.assertIn(expected_result, test_data) + + # Then, test the result of the state run when a failure is not expected to happen + test_data = state_run['cmd_|-test_non_failing_state_|-echo "Should not run"_|-run']['comment'] + expected_result = 'State was not run because onfail req did not change' + self.assertIn(expected_result, test_data) + # listen tests def test_listen_requisite(self): @@ -1124,6 +1250,21 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): listener_state = 'cmd_|-listener_test_listening_resolution_two_|-echo "Successful listen resolution"_|-mod_watch' self.assertIn(listener_state, state_run) + def test_listen_requisite_no_state_module(self): + ''' + Tests a simple state using the listen requisite + ''' + + # Only run the state once and keep the return data + state_run = self.run_function('state.sls', mods='requisites.listen_simple_no_state_module') + # First, test the result of the state run when a listener is expected to trigger + listener_state = 'cmd_|-listener_test_listening_change_state_|-echo "Listening State"_|-mod_watch' + self.assertIn(listener_state, state_run) + + # Then, test the result of the state run when a listener should not trigger + absent_state = 'cmd_|-listener_test_listening_non_changing_state_|-echo "Only run once"_|-mod_watch' + self.assertNotIn(absent_state, state_run) + def test_issue_30820_requisite_in_match_by_name(self): ''' This tests the case where a requisite_in matches by name instead of ID From f81bb61f2d51a07af62cd8d38e00faef81d6612f Mon Sep 17 00:00:00 2001 From: comsul Date: Fri, 29 Sep 2017 19:27:02 +0800 Subject: [PATCH 044/159] check_result: Correctly check the __extend__ state. https://github.com/saltstack/salt/issues/42512 check_state_result() is expecting each highstate item to be a dict, but states using extend are lists. Conflicts: - salt/utils/state.py --- salt/utils/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index ae8caf0ad35..ce19df5b81c 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -2016,7 +2016,11 @@ def check_state_result(running, recurse=False, highstate=None): ret = True for state_id, state_result in six.iteritems(running): - if not recurse and not isinstance(state_result, dict): + expected_type = dict + # The __extend__ state is a list + if "__extend__" == state_id: + expected_type = list + if not recurse and not isinstance(state_result, expected_type): ret = False if ret and isinstance(state_result, dict): result = state_result.get('result', _empty) From 75361505674aaf3856a86239f11a5ad14c151836 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Thu, 16 Nov 2017 14:12:59 -0500 Subject: [PATCH 045/159] Add service.running integration state test --- tests/integration/states/test_service.py | 42 +++++++++++++++++++----- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/tests/integration/states/test_service.py b/tests/integration/states/test_service.py index ec50bad4d8e..1426ee6edac 100644 --- a/tests/integration/states/test_service.py +++ b/tests/integration/states/test_service.py @@ -7,7 +7,6 @@ from __future__ import absolute_import # Import Salt Testing libs from tests.support.case import ModuleCase -from tests.support.unit import skipIf from tests.support.helpers import destructiveTest from tests.support.mixins import SaltReturnAssertsMixin @@ -15,32 +14,58 @@ from tests.support.mixins import SaltReturnAssertsMixin import salt.utils INIT_DELAY = 5 -SERVICE_NAME = 'crond' @destructiveTest -@skipIf(salt.utils.which('crond') is None, 'crond not installed') class ServiceTest(ModuleCase, SaltReturnAssertsMixin): ''' Validate the service state ''' + def setUp(self): + self.service_name = 'cron' + cmd_name = 'crontab' + os_family = self.run_function('grains.get', ['os_family']) + if os_family == 'RedHat': + self.service_name = 'crond' + elif os_family == 'Arch': + self.service_name = 'systemd-journald' + cmd_name = 'systemctl' + + if salt.utils.which(cmd_name) is None: + self.skipTest('{0} is not installed'.format(cmd_name)) + def check_service_status(self, exp_return): ''' helper method to check status of service ''' - check_status = self.run_function('service.status', name=SERVICE_NAME) + check_status = self.run_function('service.status', + name=self.service_name) if check_status is not exp_return: self.fail('status of service is not returning correctly') + def test_service_running(self): + ''' + test service.running state module + ''' + stop_service = self.run_function('service.stop', self.service_name) + self.assertTrue(stop_service) + self.check_service_status(False) + + start_service = self.run_state('service.running', + name=self.service_name) + self.assertTrue(start_service) + self.check_service_status(True) + def test_service_dead(self): ''' test service.dead state module ''' - start_service = self.run_state('service.running', name=SERVICE_NAME) + start_service = self.run_state('service.running', + name=self.service_name) self.assertSaltTrueReturn(start_service) self.check_service_status(True) - ret = self.run_state('service.dead', name=SERVICE_NAME) + ret = self.run_state('service.dead', name=self.service_name) self.assertSaltTrueReturn(ret) self.check_service_status(False) @@ -48,11 +73,12 @@ class ServiceTest(ModuleCase, SaltReturnAssertsMixin): ''' test service.dead state module with init_delay arg ''' - start_service = self.run_state('service.running', name=SERVICE_NAME) + start_service = self.run_state('service.running', + name=self.service_name) self.assertSaltTrueReturn(start_service) self.check_service_status(True) - ret = self.run_state('service.dead', name=SERVICE_NAME, + ret = self.run_state('service.dead', name=self.service_name, init_delay=INIT_DELAY) self.assertSaltTrueReturn(ret) self.check_service_status(False) From eb91af999ea84d6a4147ef3bcf0d1b3b25e7d49e Mon Sep 17 00:00:00 2001 From: rallytime Date: Thu, 16 Nov 2017 15:36:53 -0500 Subject: [PATCH 046/159] Remove logging from top of napalm util file Logging should not be done anywhere in a file before a module is loaded. This includes the `__virtual__()` function as well. This is because these messages will be logged on every call to the loader, which muddies the logs when the function(s) being called do not need to use that particular module. --- salt/utils/napalm.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/salt/utils/napalm.py b/salt/utils/napalm.py index 0608589a6ae..1f560fd280f 100644 --- a/salt/utils/napalm.py +++ b/salt/utils/napalm.py @@ -15,18 +15,19 @@ Utils for the NAPALM modules and proxy. .. versionadded:: 2017.7.0 ''' +# Import Python libs from __future__ import absolute_import - import traceback import logging import importlib from functools import wraps -log = logging.getLogger(__file__) -import salt.utils +# Import Salt libs +from salt.ext import six as six import salt.output +import salt.utils -# Import third party lib +# Import third party libs try: # will try to import NAPALM # https://github.com/napalm-automation/napalm @@ -36,21 +37,16 @@ try: # pylint: enable=W0611 HAS_NAPALM = True HAS_NAPALM_BASE = False # doesn't matter anymore, but needed for the logic below - log.debug('napalm seems to be installed') try: NAPALM_MAJOR = int(napalm.__version__.split('.')[0]) - log.debug('napalm version: %s', napalm.__version__) except AttributeError: NAPALM_MAJOR = 0 except ImportError: - log.info('napalm doesnt seem to be installed, trying to import napalm_base') HAS_NAPALM = False try: import napalm_base - log.debug('napalm_base seems to be installed') HAS_NAPALM_BASE = True except ImportError: - log.info('napalm_base doesnt seem to be installed either') HAS_NAPALM_BASE = False try: @@ -62,7 +58,7 @@ try: except ImportError: HAS_CONN_CLOSED_EXC_CLASS = False -from salt.ext import six as six +log = logging.getLogger(__file__) def is_proxy(opts): @@ -98,8 +94,6 @@ def virtual(opts, virtualname, filename): Returns the __virtual__. ''' if ((HAS_NAPALM and NAPALM_MAJOR >= 2) or HAS_NAPALM_BASE) and (is_proxy(opts) or is_minion(opts)): - if HAS_NAPALM_BASE: - log.info('You still seem to use napalm_base. Please consider upgrading to napalm >= 2.0.0') return virtualname else: return ( From dea335e3d02abe72ddcc8aca002c1c2fa7f040c3 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 16 Nov 2017 08:50:26 -0800 Subject: [PATCH 047/159] Adding state files used by new integration tests. --- .../listen_simple_no_state_module.sls | 36 ++++++++++++ .../onchanges_simple_no_state_module.sls | 21 +++++++ .../onfail_simple_no_state_module.sls | 19 ++++++ .../prereq_simple_no_state_module.sls | 33 +++++++++++ .../requisites/require_no_state_module.sls | 58 +++++++++++++++++++ .../base/requisites/use_no_state_module.sls | 55 ++++++++++++++++++ 6 files changed, 222 insertions(+) create mode 100644 tests/integration/files/file/base/requisites/listen_simple_no_state_module.sls create mode 100644 tests/integration/files/file/base/requisites/onchanges_simple_no_state_module.sls create mode 100644 tests/integration/files/file/base/requisites/onfail_simple_no_state_module.sls create mode 100644 tests/integration/files/file/base/requisites/prereq_simple_no_state_module.sls create mode 100644 tests/integration/files/file/base/requisites/require_no_state_module.sls create mode 100644 tests/integration/files/file/base/requisites/use_no_state_module.sls diff --git a/tests/integration/files/file/base/requisites/listen_simple_no_state_module.sls b/tests/integration/files/file/base/requisites/listen_simple_no_state_module.sls new file mode 100644 index 00000000000..cba87592f5a --- /dev/null +++ b/tests/integration/files/file/base/requisites/listen_simple_no_state_module.sls @@ -0,0 +1,36 @@ +successful_changing_state: + cmd.run: + - name: echo "Successful Change" + +# mock is installed with salttesting, so it should already be +# present on the system, resulting in no changes +non_changing_state: + pip.installed: + - name: mock + +test_listening_change_state: + cmd.run: + - name: echo "Listening State" + - listen: + - successful_changing_state + +test_listening_non_changing_state: + cmd.run: + - name: echo "Only run once" + - listen: + - non_changing_state + +# test that requisite resolution for listen uses ID declaration. +# test_listening_resolution_one and test_listening_resolution_two +# should both run. +test_listening_resolution_one: + cmd.run: + - name: echo "Successful listen resolution" + - listen: + - successful_changing_state + +test_listening_resolution_two: + cmd.run: + - name: echo "Successful listen resolution" + - listen: + - successful_changing_state diff --git a/tests/integration/files/file/base/requisites/onchanges_simple_no_state_module.sls b/tests/integration/files/file/base/requisites/onchanges_simple_no_state_module.sls new file mode 100644 index 00000000000..a9dc505a950 --- /dev/null +++ b/tests/integration/files/file/base/requisites/onchanges_simple_no_state_module.sls @@ -0,0 +1,21 @@ +changing_state: + cmd.run: + - name: echo "Changed!" + +# mock is installed with salttesting, so it should already be +# present on the system, resulting in no changes +non_changing_state: + pip.installed: + - name: mock + +test_changing_state: + cmd.run: + - name: echo "Success!" + - onchanges: + - changing_state + +test_non_changing_state: + cmd.run: + - name: echo "Should not run" + - onchanges: + - non_changing_state diff --git a/tests/integration/files/file/base/requisites/onfail_simple_no_state_module.sls b/tests/integration/files/file/base/requisites/onfail_simple_no_state_module.sls new file mode 100644 index 00000000000..3ba67a5ec22 --- /dev/null +++ b/tests/integration/files/file/base/requisites/onfail_simple_no_state_module.sls @@ -0,0 +1,19 @@ +failing_state: + cmd.run: + - name: asdf + +non_failing_state: + cmd.run: + - name: echo "Non-failing state" + +test_failing_state: + cmd.run: + - name: echo "Success!" + - onfail: + - failing_state + +test_non_failing_state: + cmd.run: + - name: echo "Should not run" + - onfail: + - non_failing_state diff --git a/tests/integration/files/file/base/requisites/prereq_simple_no_state_module.sls b/tests/integration/files/file/base/requisites/prereq_simple_no_state_module.sls new file mode 100644 index 00000000000..bcfb8c02c70 --- /dev/null +++ b/tests/integration/files/file/base/requisites/prereq_simple_no_state_module.sls @@ -0,0 +1,33 @@ +# B --+ +# | +# C <-+ ----+ +# | +# A <-------+ + +# runs after C +A: + cmd.run: + - name: echo A third + # is running in test mode before C + # C gets executed first if this states modify something + - prereq_in: + - C + +# runs before C +B: + cmd.run: + - name: echo B first + # will test C and be applied only if C changes, + # and then will run before C + - prereq: + - C +C: + cmd.run: + - name: echo C second + +# will fail with "The following requisites were not found" +I: + cmd.run: + - name: echo I + - prereq: + - Z diff --git a/tests/integration/files/file/base/requisites/require_no_state_module.sls b/tests/integration/files/file/base/requisites/require_no_state_module.sls new file mode 100644 index 00000000000..234fe981e44 --- /dev/null +++ b/tests/integration/files/file/base/requisites/require_no_state_module.sls @@ -0,0 +1,58 @@ +# Complex require/require_in graph +# +# Relative order of C>E is given by the definition order +# +# D (1) <--+ +# | +# B (2) ---+ <-+ <-+ <-+ +# | | | +# C (3) <--+ --|---|---+ +# | | | +# E (4) ---|---|---+ <-+ +# | | | +# A (5) ---+ --+ ------+ +# + +A: + cmd.run: + - name: echo A fifth + - require: + - C +B: + cmd.run: + - name: echo B second + - require_in: + - A + - C + +C: + cmd.run: + - name: echo C third + +D: + cmd.run: + - name: echo D first + - require_in: + - B + +E: + cmd.run: + - name: echo E fourth + - require: + - B + - require_in: + - A + +# will fail with "The following requisites were not found" +G: + cmd.run: + - name: echo G + - require: + - Z +# will fail with "The following requisites were not found" +H: + cmd.run: + - name: echo H + - require: + - Z + diff --git a/tests/integration/files/file/base/requisites/use_no_state_module.sls b/tests/integration/files/file/base/requisites/use_no_state_module.sls new file mode 100644 index 00000000000..f055980bd4d --- /dev/null +++ b/tests/integration/files/file/base/requisites/use_no_state_module.sls @@ -0,0 +1,55 @@ +# None of theses states should run +A: + cmd.run: + - name: echo "A" + - onlyif: 'false' + +# issue #8235 +#B: +# cmd.run: +# - name: echo "B" +# # here used without "-" +# - use: +# cmd: A + +C: + cmd.run: + - name: echo "C" + - use: + - A + +D: + cmd.run: + - name: echo "D" + - onlyif: 'false' + - use_in: + - E + +E: + cmd.run: + - name: echo "E" + +# issue 8235 +#F: +# cmd.run: +# - name: echo "F" +# - onlyif: return 0 +# - use_in: +# cmd: G +# +#G: +# cmd.run: +# - name: echo "G" + +# issue xxxx +#H: +# cmd.run: +# - name: echo "H" +# - use: +# - cmd: C +#I: +# cmd.run: +# - name: echo "I" +# - use: +# - cmd: E + From 0a70c673ac5a6c0552d3806e93935f4a5b74d282 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 16 Nov 2017 13:26:19 -0800 Subject: [PATCH 048/159] Swapping out the code in call_listen to look in chunks to gather requisites instead of high, which is not always available when call_listen is called. --- salt/state.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/salt/state.py b/salt/state.py index 2c2c4258a98..42801d83d72 100644 --- a/salt/state.py +++ b/salt/state.py @@ -2575,7 +2575,7 @@ class State(object): self.event(running[tag], len(chunks), fire_event=low.get(u'fire_event')) return running - def call_listen(self, chunks, running, high): + def call_listen(self, chunks, running): ''' Find all of the listen routines and call the associated mod_watch runs ''' @@ -2596,22 +2596,12 @@ class State(object): for key, val in six.iteritems(l_dict): for listen_to in val: if not isinstance(listen_to, dict): - if listen_to in high: - _listen_high = [x for x - in high[listen_to] - if not x.startswith('__')] - listen_to = {_listen_high[0]: listen_to} - else: - found = False - for _id in iter(high): - for state in [state for state - in iter(high[_id]) - if not state.startswith('__')]: - for j in iter(high[_id][state]): - if isinstance(j, dict) and 'name' in j: - if j['name'] == listen_to: - listen_to = {state: _id} - found = True + found = False + for chunk in chunks: + if chunk['__id__'] == listen_to or \ + chunk['name'] == listen_to: + listen_to = {chunk['state']: chunk['__id__']} + found = True if not found: continue for lkey, lval in six.iteritems(listen_to): @@ -2679,7 +2669,7 @@ class State(object): if errors: return errors ret = self.call_chunks(chunks) - ret = self.call_listen(chunks, ret, high) + ret = self.call_listen(chunks, ret) def _cleanup_accumulator_data(): accum_data_path = os.path.join( From 2c937dc14ec8ef85573370e74d22544ab2409597 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 16 Nov 2017 14:05:23 -0800 Subject: [PATCH 049/159] Moving an if statement outside the for loop over the list of chunks. --- salt/state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/state.py b/salt/state.py index 42801d83d72..f0e405e77d0 100644 --- a/salt/state.py +++ b/salt/state.py @@ -2602,8 +2602,8 @@ class State(object): chunk['name'] == listen_to: listen_to = {chunk['state']: chunk['__id__']} found = True - if not found: - continue + if not found: + continue for lkey, lval in six.iteritems(listen_to): if (lkey, lval) not in crefs: rerror = {_l_tag(lkey, lval): From 3d2f30505392e69a0ebe411c3a75008765632432 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 16 Nov 2017 16:02:56 -0800 Subject: [PATCH 050/159] Fixing expected test results. --- tests/integration/modules/test_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/modules/test_state.py b/tests/integration/modules/test_state.py index 8f0b64a9c2e..6e2d8aaba61 100644 --- a/tests/integration/modules/test_state.py +++ b/tests/integration/modules/test_state.py @@ -1263,7 +1263,7 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_function('state.sls', mods='requisites.use_no_state_module') self.assertReturnNonEmptySaltType(ret) for item, descr in six.iteritems(ret): - self.assertEqual(descr['comment'], 'onlyif execution failed') + self.assertEqual(descr['comment'], 'onlyif condition is false') def test_get_file_from_env_in_top_match(self): tgt = os.path.join(TMP, 'prod-cheese-file') From d50b7e0062f88a04789ddfef3b7fee62bd3f8dc1 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 17 Nov 2017 10:34:30 -0500 Subject: [PATCH 051/159] Handle deprecation path for pki_dir in verify_env util In PR #43474, a new option was added named "sensitive_dirs". This option was intended to replace "pki_dir". However, we don't want to drop support for pki_dir out from under people so we need to put pki_dir on a deprecation path. I have also moved the new sensitive_dirs option to the _end_ of the list of kwargs. That way, if other users are using positional kwargs, we won't break those references either. People will have time to adjust their calls to verify_env when they see the warning. --- salt/utils/verify.py | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/salt/utils/verify.py b/salt/utils/verify.py index 7c1f35e4395..77ae50a7e5e 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -30,6 +30,7 @@ import salt.defaults.exitcodes import salt.utils.files import salt.utils.platform import salt.utils.user +import salt.utils.versions log = logging.getLogger(__name__) @@ -194,13 +195,32 @@ def verify_files(files, user): return True -def verify_env(dirs, user, permissive=False, sensitive_dirs=None, skip_extra=False): +def verify_env( + dirs, + user, + permissive=False, + pki_dir='', + skip_extra=False, + sensitive_dirs=None): ''' Verify that the named directories are in place and that the environment can shake the salt ''' + if pki_dir: + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'pki_dir\' was detected: \'pki_dir\' has been deprecated ' + 'in favor of \'sensitive_dirs\'. Support for \'pki_dir\' will be ' + 'removed in Salt Neon.' + ) + sensitive_dirs = sensitive_dirs or [] + sensitive_dirs.append(list(pki_dir)) + if salt.utils.platform.is_windows(): - return win_verify_env(dirs, permissive, sensitive_dirs, skip_extra) + return win_verify_env(dirs, + permissive=permissive, + skip_extra=skip_extra, + sensitive_dirs=sensitive_dirs) import pwd # after confirming not running Windows try: pwnam = pwd.getpwnam(user) @@ -526,11 +546,26 @@ def verify_log(opts): log.warning('Insecure logging configuration detected! Sensitive data may be logged.') -def win_verify_env(dirs, permissive=False, sensitive_dirs=None, skip_extra=False): +def win_verify_env( + dirs, + permissive=False, + pki_dir='', + skip_extra=False, + sensitive_dirs=None): ''' Verify that the named directories are in place and that the environment can shake the salt ''' + if pki_dir: + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'pki_dir\' was detected: \'pki_dir\' has been deprecated ' + 'in favor of \'sensitive_dirs\'. Support for \'pki_dir\' will be ' + 'removed in Salt Neon.' + ) + sensitive_dirs = sensitive_dirs or [] + sensitive_dirs.append(list(pki_dir)) + import salt.utils.win_functions import salt.utils.win_dacl From 361205d6c52f08e387e99f10ba7c0132809a4adc Mon Sep 17 00:00:00 2001 From: skjaro Date: Fri, 17 Nov 2017 22:43:07 +0100 Subject: [PATCH 052/159] add new set types hash:mac and hash:ip,mark and create option skbinfo Fixes state discarding options with entries --- salt/modules/ipset.py | 97 +++++++++++++++++++++++++------------------ salt/states/ipset.py | 4 +- 2 files changed, 59 insertions(+), 42 deletions(-) diff --git a/salt/modules/ipset.py b/salt/modules/ipset.py index d67c43d78be..8b1f0189b9b 100644 --- a/salt/modules/ipset.py +++ b/salt/modules/ipset.py @@ -36,11 +36,12 @@ _IPSET_FAMILIES = { 'ip6': 'inet6', } -_IPSET_SET_TYPES = [ +_IPSET_SET_TYPES = set([ 'bitmap:ip', 'bitmap:ip,mac', 'bitmap:port', 'hash:ip', + 'hash:mac', 'hash:ip,port', 'hash:ip,port,ip', 'hash:ip,port,net', @@ -49,32 +50,37 @@ _IPSET_SET_TYPES = [ 'hash:net,iface', 'hash:net,port', 'hash:net,port,net', + 'hash:ip,mark', 'list:set' - ] + ]) _CREATE_OPTIONS = { - 'bitmap:ip': ['range', 'netmask', 'timeout', 'counters', 'comment'], - 'bitmap:ip,mac': ['range', 'timeout', 'counters', 'comment'], - 'bitmap:port': ['range', 'timeout', 'counters', 'comment'], - 'hash:ip': ['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment'], - 'hash:net': ['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment'], - 'hash:net,net': ['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment'], - 'hash:net,port': ['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment'], - 'hash:net,port,net': ['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment'], - 'hash:ip,port,ip': ['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment'], - 'hash:ip,port,net': ['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment'], - 'hash:ip,port': ['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment'], - 'hash:net,iface': ['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment'], - 'list:set': ['size', 'timeout', 'counters', 'comment'], + 'bitmap:ip': set(['range', 'netmask', 'timeout', 'counters', 'comment', 'skbinfo']), + 'bitmap:ip,mac': set(['range', 'timeout', 'counters', 'comment', 'skbinfo']), + 'bitmap:port': set(['range', 'timeout', 'counters', 'comment', 'skbinfo']), + 'hash:ip': set(['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment', 'skbinfo']), + 'hash:mac': set(['hashsize', 'maxelem', 'timeout', 'counters', 'comment', 'skbinfo']), + 'hash:net': set(['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment', 'skbinfo']), + 'hash:net,net': set(['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment', 'skbinfo']), + 'hash:net,port': set(['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment', 'skbinfo']), + 'hash:net,port,net': set(['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment', 'skbinfo']), + 'hash:ip,port,ip': set(['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment', 'skbinfo']), + 'hash:ip,port,net': set(['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment', 'skbinfo']), + 'hash:ip,port': set(['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment', 'skbinfo']), + 'hash:ip,mark': set(['family', 'markmask', 'hashsize', 'maxelem', 'timeout', 'counters', 'comment', 'skbinfo']), + 'hash:net,iface': set(['family', 'hashsize', 'maxelem', 'netmask', 'timeout', 'counters', 'comment', 'skbinfo']), + 'list:set': set(['size', 'timeout', 'counters', 'comment']), } +_CREATE_OPTIONS_WITHOUT_VALUE = set(['comment', 'counters', 'skbinfo']) _CREATE_OPTIONS_REQUIRED = { 'bitmap:ip': ['range'], 'bitmap:ip,mac': ['range'], 'bitmap:port': ['range'], 'hash:ip': [], + 'hash:mac': [], 'hash:net': [], 'hash:net,net': [], 'hash:ip,port': [], @@ -83,24 +89,27 @@ _CREATE_OPTIONS_REQUIRED = { 'hash:ip,port,net': [], 'hash:net,port,net': [], 'hash:net,iface': [], + 'hash:ip,mark': [], 'list:set': [] } _ADD_OPTIONS = { - 'bitmap:ip': ['timeout', 'packets', 'bytes'], - 'bitmap:ip,mac': ['timeout', 'packets', 'bytes'], - 'bitmap:port': ['timeout', 'packets', 'bytes'], - 'hash:ip': ['timeout', 'packets', 'bytes'], - 'hash:net': ['timeout', 'nomatch', 'packets', 'bytes'], - 'hash:net,net': ['timeout', 'nomatch', 'packets', 'bytes'], - 'hash:net,port': ['timeout', 'nomatch', 'packets', 'bytes'], - 'hash:net,port,net': ['timeout', 'nomatch', 'packets', 'bytes'], - 'hash:ip,port,ip': ['timeout', 'packets', 'bytes'], - 'hash:ip,port,net': ['timeout', 'nomatch', 'packets', 'bytes'], - 'hash:ip,port': ['timeout', 'nomatch', 'packets', 'bytes'], - 'hash:net,iface': ['timeout', 'nomatch', 'packets', 'bytes'], - 'list:set': ['timeout', 'packets', 'bytes'], + 'bitmap:ip': set(['timeout', 'packets', 'bytes', 'skbmark', 'skbprio', 'skbqueue']), + 'bitmap:ip,mac': set(['timeout', 'packets', 'bytes', 'skbmark', 'skbprio', 'skbqueue']), + 'bitmap:port': set(['timeout', 'packets', 'bytes', 'skbmark', 'skbprio', 'skbprio']), + 'hash:ip': set(['timeout', 'packets', 'bytes', 'skbmark', 'skbprio', 'skbqueue']), + 'hash:mac': set(['timeout', 'packets', 'bytes', 'skbmark', 'skbprio', 'skbqueue']), + 'hash:net': set(['timeout', 'nomatch', 'packets', 'bytes', 'skbmark', 'skbprio', 'skbqueue']), + 'hash:net,net': set(['timeout', 'nomatch', 'packets', 'bytes', 'skbmark', 'skbprio', 'skbqueue']), + 'hash:net,port': set(['timeout', 'nomatch', 'packets', 'bytes', 'skbmark', 'skbprio', 'skbqueue']), + 'hash:net,port,net': set(['timeout', 'nomatch', 'packets', 'bytes', 'skbmark', 'skbprio', 'skbqueue']), + 'hash:ip,port,ip': set(['timeout', 'packets', 'bytes', 'skbmark', 'skbprio', 'skbqueue']), + 'hash:ip,port,net': set(['timeout', 'nomatch', 'packets', 'bytes', 'skbmark', 'skbprio', 'skbqueue']), + 'hash:ip,port': set(['timeout', 'nomatch', 'packets', 'bytes', 'skbmark', 'skbprio', 'skbqueue']), + 'hash:net,iface': set(['timeout', 'nomatch', 'packets', 'bytes', 'skbmark', 'skbprio', 'skbqueue']), + 'hash:ip,mark': set(['timeout', 'packets', 'bytes', 'skbmark', 'skbprio', 'skbqueue']), + 'list:set': set(['timeout', 'packets', 'bytes', 'skbmark', 'skbprio', 'skbqueue']), } @@ -173,7 +182,10 @@ def new_set(set=None, set_type=None, family='ipv4', comment=False, **kwargs): for item in _CREATE_OPTIONS[set_type]: if item in kwargs: - cmd = '{0} {1} {2} '.format(cmd, item, kwargs[item]) + if item in _CREATE_OPTIONS_WITHOUT_VALUE: + cmd = '{0} {1} '.format(cmd, item) + else: + cmd = '{0} {1} {2} '.format(cmd, item, kwargs[item]) # Family only valid for certain set types if 'family' in _CREATE_OPTIONS[set_type]: @@ -307,7 +319,7 @@ def check_set(set=None, family='ipv4'): return True -def add(set=None, entry=None, family='ipv4', **kwargs): +def add(setname=None, entry=None, family='ipv4', **kwargs): ''' Append an entry to the specified set. @@ -320,14 +332,14 @@ def add(set=None, entry=None, family='ipv4', **kwargs): salt '*' ipset.add setname 192.168.0.3,AA:BB:CC:DD:EE:FF ''' - if not set: + if not setname: return 'Error: Set needs to be specified' if not entry: return 'Error: Entry needs to be specified' - setinfo = _find_set_info(set) + setinfo = _find_set_info(setname) if not setinfo: - return 'Error: Set {0} does not exist'.format(set) + return 'Error: Set {0} does not exist'.format(setname) settype = setinfo['Type'] @@ -335,27 +347,32 @@ def add(set=None, entry=None, family='ipv4', **kwargs): if 'timeout' in kwargs: if 'timeout' not in setinfo['Header']: - return 'Error: Set {0} not created with timeout support'.format(set) + return 'Error: Set {0} not created with timeout support'.format(setname) if 'packets' in kwargs or 'bytes' in kwargs: if 'counters' not in setinfo['Header']: - return 'Error: Set {0} not created with counters support'.format(set) + return 'Error: Set {0} not created with counters support'.format(setname) if 'comment' in kwargs: if 'comment' not in setinfo['Header']: - return 'Error: Set {0} not created with comment support'.format(set) - cmd = '{0} comment "{1}"'.format(cmd, kwargs['comment']) + return 'Error: Set {0} not created with comment support'.format(setname) + if 'comment' not in entry: + cmd = '{0} comment "{1}"'.format(cmd, kwargs['comment']) + + if len(set(['skbmark', 'skbprio', 'skbqueue']) & set(kwargs.keys())) > 0: + if 'skbinfo' not in setinfo['Header']: + return 'Error: Set {0} not created with skbinfo support'.format(setname) for item in _ADD_OPTIONS[settype]: if item in kwargs: cmd = '{0} {1} {2}'.format(cmd, item, kwargs[item]) - current_members = _find_set_members(set) + current_members = _find_set_members(setname) if cmd in current_members: - return 'Warn: Entry {0} already exists in set {1}'.format(cmd, set) + return 'Warn: Entry {0} already exists in set {1}'.format(cmd, setname) # Using -exist to ensure entries are updated if the comment changes - cmd = '{0} add -exist {1} {2}'.format(_ipset_cmd(), set, cmd) + cmd = '{0} add -exist {1} {2}'.format(_ipset_cmd(), setname, cmd) out = __salt__['cmd.run'](cmd, python_shell=False) if len(out) == 0: diff --git a/salt/states/ipset.py b/salt/states/ipset.py index 27f4e243feb..d3d2af37f77 100644 --- a/salt/states/ipset.py +++ b/salt/states/ipset.py @@ -221,7 +221,7 @@ def present(name, entry=None, family='ipv4', **kwargs): kwargs['set_name'], family) else: - command = __salt__['ipset.add'](kwargs['set_name'], entry, family, **kwargs) + command = __salt__['ipset.add'](kwargs['set_name'], _entry, family, **kwargs) if 'Error' not in command: ret['changes'] = {'locale': name} ret['comment'] += 'entry {0} added to set {1} for family {2}\n'.format( @@ -293,7 +293,7 @@ def absent(name, entry=None, entries=None, family='ipv4', **kwargs): kwargs['set_name'], family) else: - command = __salt__['ipset.delete'](kwargs['set_name'], entry, family, **kwargs) + command = __salt__['ipset.delete'](kwargs['set_name'], _entry, family, **kwargs) if 'Error' not in command: ret['changes'] = {'locale': name} ret['result'] = True From f5dbab21caca87ce6f98c0f9a27501702ef71126 Mon Sep 17 00:00:00 2001 From: skjaro Date: Sat, 18 Nov 2017 18:36:46 +0100 Subject: [PATCH 053/159] Fix multiple spaces on entry options start --- salt/states/ipset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/ipset.py b/salt/states/ipset.py index d3d2af37f77..194fa5172cb 100644 --- a/salt/states/ipset.py +++ b/salt/states/ipset.py @@ -204,7 +204,7 @@ def present(name, entry=None, family='ipv4', **kwargs): entry_opts = 'timeout {0} {1}'.format(kwargs['timeout'], entry_opts) if 'comment' in kwargs and 'comment' not in entry_opts: entry_opts = '{0} comment "{1}"'.format(entry_opts, kwargs['comment']) - _entry = ' '.join([entry, entry_opts]).strip() + _entry = ' '.join([entry, entry_opts.lstrip()]).strip() if __salt__['ipset.check'](kwargs['set_name'], _entry, From 4209d2604c3f5ba9623de60f12f7dbf23cf02bb9 Mon Sep 17 00:00:00 2001 From: PeterS242 Date: Sat, 18 Nov 2017 21:25:23 -0500 Subject: [PATCH 054/159] Fixup on yumpkgs.py to better handle YUM Extra Options (addressing issue https://github.com/saltstack/salt/issues/44590) --- salt/modules/yumpkg.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index d7d28276e63..bd7a5d32cfb 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -279,11 +279,23 @@ def _get_extra_options(**kwargs): ''' ret = [] kwargs = salt.utils.args.clean_kwargs(**kwargs) + + # Remove already handled options from kwargs + fromrepo = kwargs.pop('fromrepo', '') + repo = kwargs.pop('repo', '') + disablerepo = kwargs.pop('disablerepo', '') + enablerepo = kwargs.pop('enablerepo', '') + disable_excludes = kwargs.pop('disableexcludes', '') + branch = kwargs.pop('branch', '') + for key, value in six.iteritems(kwargs): - if isinstance(key, six.string_types): + if isinstance(value, six.string_types): + log.info('Adding extra option --%s=\'%s\'', key, value) ret.append('--{0}=\'{1}\''.format(key, value)) elif value is True: + log.info('Adding extra option --%s', key) ret.append('--{0}'.format(key)) + log.info('Adding extra options %s', ret) return ret From 97ac9d07934471a653b9395c2475be4df6f209df Mon Sep 17 00:00:00 2001 From: Jorge Schrauwen Date: Sun, 19 Nov 2017 11:03:04 +0100 Subject: [PATCH 055/159] The hint to use force is not stdout but also stderr, filter that out for labelclear --- salt/modules/zpool.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/salt/modules/zpool.py b/salt/modules/zpool.py index 93f8204e5bd..2a9f80f6636 100644 --- a/salt/modules/zpool.py +++ b/salt/modules/zpool.py @@ -1375,7 +1375,13 @@ def labelclear(device, force=False): # Bring all specified devices offline res = __salt__['cmd.run_all'](cmd, python_shell=False) if res['retcode'] != 0: - ret[device] = res['stderr'] if 'stderr' in res else res['stdout'] + ## NOTE: skip the "use '-f' hint" + res['stderr'] = res['stderr'].split("\n") + if len(res['stderr']) >= 1: + if res['stderr'][0].startswith("use '-f'"): + del res['stderr'][0] + res['stderr'] = "\n".join(res['stderr']) + ret[device] = res['stderr'] if 'stderr' in res and res['stderr'] else res['stdout'] else: ret[device] = 'cleared' return ret From 4829157d3df7a4c82cb5a9709be97794be3ac354 Mon Sep 17 00:00:00 2001 From: Jorge Schrauwen Date: Sun, 19 Nov 2017 12:03:31 +0100 Subject: [PATCH 056/159] Switch from exists() to __salt__['zpool.exists'] To make writing testing easier, make internal calls to exists() use the salt calling interface. --- salt/modules/zpool.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/salt/modules/zpool.py b/salt/modules/zpool.py index 2a9f80f6636..584e71f5b3b 100644 --- a/salt/modules/zpool.py +++ b/salt/modules/zpool.py @@ -509,7 +509,7 @@ def destroy(zpool, force=False): ''' ret = {} ret[zpool] = {} - if not exists(zpool): + if not __salt__['zpool.exists'](zpool): ret[zpool] = 'storage pool does not exist' else: zpool_cmd = _check_zpool() @@ -555,7 +555,7 @@ def scrub(zpool, stop=False, pause=False): ''' ret = {} ret[zpool] = {} - if exists(zpool): + if __salt__['zpool.exists'](zpool): zpool_cmd = _check_zpool() if stop: action = '-s ' @@ -651,7 +651,7 @@ def create(zpool, *vdevs, **kwargs): ret = {} # Check if the pool_name is already being used - if exists(zpool): + if __salt__['zpool.exists'](zpool): ret[zpool] = 'storage pool already exists' return ret @@ -741,7 +741,7 @@ def add(zpool, *vdevs, **kwargs): ret = {} # check for pool - if not exists(zpool): + if not __salt__['zpool.exists'](zpool): ret[zpool] = 'storage pool does not exist' return ret @@ -794,7 +794,7 @@ def attach(zpool, device, new_device, force=False): dlist = [] # check for pool - if not exists(zpool): + if not __salt__['zpool.exists'](zpool): ret[zpool] = 'storage pool does not exist' return ret @@ -856,7 +856,7 @@ def detach(zpool, device): dlist = [] # check for pool - if not exists(zpool): + if not __salt__['zpool.exists'](zpool): ret[zpool] = 'storage pool does not exist' return ret @@ -923,11 +923,11 @@ def split(zpool, newzpool, **kwargs): ret = {} # Check if the pool_name is already being used - if exists(newzpool): + if __salt__['zpool.exists'](newzpool): ret[newzpool] = 'storage pool already exists' return ret - if not exists(zpool): + if not __salt__['zpool.exists'](zpool): ret[zpool] = 'storage pool does not exists' return ret @@ -996,7 +996,7 @@ def replace(zpool, old_device, new_device=None, force=False): ''' ret = {} # Make sure pool is there - if not exists(zpool): + if not __salt__['zpool.exists'](zpool): ret[zpool] = 'storage pool does not exist' return ret @@ -1109,7 +1109,7 @@ def export(*pools, **kwargs): return ret for pool in pools: - if not exists(pool): + if not __salt__['zpool.exists'](pool): ret[pool] = 'storage pool does not exist' else: pool_present.append(pool) @@ -1224,7 +1224,7 @@ def import_(zpool=None, new_name=None, **kwargs): ret['error'] = res['stderr'] if 'stderr' in res else res['stdout'] else: if zpool: - ret[zpool if not new_name else new_name] = 'imported' if exists(zpool if not new_name else new_name) else 'not found' + ret[zpool if not new_name else new_name] = 'imported' if __salt__['zpool.exists'](zpool if not new_name else new_name) else 'not found' else: ret = True return ret @@ -1259,7 +1259,7 @@ def online(zpool, *vdevs, **kwargs): dlist = [] # Check if the pool_name exists - if not exists(zpool): + if not __salt__['zpool.exists'](zpool): ret[zpool] = 'storage pool does not exist' return ret @@ -1315,7 +1315,7 @@ def offline(zpool, *vdevs, **kwargs): ret = {} # Check if the pool_name exists - if not exists(zpool): + if not __salt__['zpool.exists'](zpool): ret[zpool] = 'storage pool does not exist' return ret From 64e0b45e513a0ceafc9ab802c49fc86756d4ebc1 Mon Sep 17 00:00:00 2001 From: Jorge Schrauwen Date: Sun, 19 Nov 2017 12:09:36 +0100 Subject: [PATCH 057/159] Add tests for zpool.labelclear and zpool.split --- tests/unit/modules/test_zpool.py | 118 +++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/tests/unit/modules/test_zpool.py b/tests/unit/modules/test_zpool.py index 871500fb994..1bb5cc754ba 100644 --- a/tests/unit/modules/test_zpool.py +++ b/tests/unit/modules/test_zpool.py @@ -13,6 +13,7 @@ from __future__ import absolute_import from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import skipIf, TestCase from tests.support.mock import ( + Mock, MagicMock, patch, NO_MOCK, @@ -152,3 +153,120 @@ class ZpoolTestCase(TestCase, LoaderModuleMockMixin): ret = zpool.get('mypool', 'readonly') res = OrderedDict([('mypool', OrderedDict([('readonly', 'off')]))]) self.assertEqual(res, ret) + + def test_split_success(self): + ''' + Tests split on success + ''' + ret = {} + ret['stdout'] = "" + ret['stderr'] = "" + ret['retcode'] = 0 + mock_cmd = MagicMock(return_value=ret) + mock_exists = Mock() + mock_exists.side_effect = [False, True] + + with patch.dict(zpool.__salt__, {'zpool.exists': mock_exists}): + with patch.dict(zpool.__salt__, {'cmd.run_all': mock_cmd}): + ret = zpool.split('datapool', 'backuppool') + res = OrderedDict([('backuppool', 'split off from datapool')]) + self.assertEqual(res, ret) + + def test_split_exist_new(self): + ''' + Tests split on exising new pool + ''' + ret = {} + ret['stdout'] = "" + ret['stderr'] = "" + ret['retcode'] = 0 + mock_cmd = MagicMock(return_value=ret) + mock_exists = Mock() + mock_exists.side_effect = [True, True] + + with patch.dict(zpool.__salt__, {'zpool.exists': mock_exists}): + with patch.dict(zpool.__salt__, {'cmd.run_all': mock_cmd}): + ret = zpool.split('datapool', 'backuppool') + res = OrderedDict([('backuppool', 'storage pool already exists')]) + self.assertEqual(res, ret) + + def test_split_missing_pool(self): + ''' + Tests split on missing source pool + ''' + ret = {} + ret['stdout'] = "" + ret['stderr'] = "" + ret['retcode'] = 0 + mock_cmd = MagicMock(return_value=ret) + mock_exists = Mock() + mock_exists.side_effect = [False, False] + + with patch.dict(zpool.__salt__, {'zpool.exists': mock_exists}): + with patch.dict(zpool.__salt__, {'cmd.run_all': mock_cmd}): + ret = zpool.split('datapool', 'backuppool') + res = OrderedDict([('datapool', 'storage pool does not exists')]) + self.assertEqual(res, ret) + + def test_split_not_mirror(self): + ''' + Tests split on source pool is not a mirror + ''' + ret = {} + ret['stdout'] = "" + ret['stderr'] = "Unable to split datapool: Source pool must be composed only of mirrors" + ret['retcode'] = 1 + mock_cmd = MagicMock(return_value=ret) + mock_exists = Mock() + mock_exists.side_effect = [False, True] + + with patch.dict(zpool.__salt__, {'zpool.exists': mock_exists}): + with patch.dict(zpool.__salt__, {'cmd.run_all': mock_cmd}): + ret = zpool.split('datapool', 'backuppool') + res = OrderedDict([('backuppool', 'Unable to split datapool: Source pool must be composed only of mirrors')]) + self.assertEqual(res, ret) + + def test_labelclear_success(self): + ''' + Tests labelclear on succesful label removal + ''' + ret = {} + ret['stdout'] = "" + ret['stderr'] = "" + ret['retcode'] = 0 + mock_cmd = MagicMock(return_value=ret) + with patch.dict(zpool.__salt__, {'cmd.run_all': mock_cmd}): + ret = zpool.labelclear('/dev/rdsk/c0t0d0', force=False) + res = OrderedDict([('/dev/rdsk/c0t0d0', 'cleared')]) + self.assertEqual(res, ret) + + def test_labelclear_cleared(self): + ''' + Tests labelclear on device with no label + ''' + ret = {} + ret['stdout'] = "" + ret['stderr'] = "failed to read label from /dev/rdsk/c0t0d0" + ret['retcode'] = 1 + mock_cmd = MagicMock(return_value=ret) + with patch.dict(zpool.__salt__, {'cmd.run_all': mock_cmd}): + ret = zpool.labelclear('/dev/rdsk/c0t0d0', force=False) + res = OrderedDict([('/dev/rdsk/c0t0d0', 'failed to read label from /dev/rdsk/c0t0d0')]) + self.assertEqual(res, ret) + + def test_labelclear_exported(self): + ''' + Tests labelclear on device with from exported pool + ''' + ret = {} + ret['stdout'] = "" + ret['stderr'] = "\n".join([ + "use '-f' to override the following error:", + '/dev/rdsk/c0t0d0 is a member of exported pool "mypool"', + ]) + ret['retcode'] = 1 + mock_cmd = MagicMock(return_value=ret) + with patch.dict(zpool.__salt__, {'cmd.run_all': mock_cmd}): + ret = zpool.labelclear('/dev/rdsk/c0t0d0', force=False) + res = OrderedDict([('/dev/rdsk/c0t0d0', '/dev/rdsk/c0t0d0 is a member of exported pool "mypool"')]) + self.assertEqual(res, ret) From 409acc1b1b4e3123414dfb3006c5ffc72183e44e Mon Sep 17 00:00:00 2001 From: Jorge Schrauwen Date: Sun, 19 Nov 2017 12:16:29 +0100 Subject: [PATCH 058/159] Also add tests for zpool.scrub --- tests/unit/modules/test_zpool.py | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/unit/modules/test_zpool.py b/tests/unit/modules/test_zpool.py index 1bb5cc754ba..d05e781100a 100644 --- a/tests/unit/modules/test_zpool.py +++ b/tests/unit/modules/test_zpool.py @@ -154,6 +154,57 @@ class ZpoolTestCase(TestCase, LoaderModuleMockMixin): res = OrderedDict([('mypool', OrderedDict([('readonly', 'off')]))]) self.assertEqual(res, ret) + def test_scrub_start(self): + ''' + Tests start of scrub + ''' + ret = {} + ret['stdout'] = "" + ret['stderr'] = "" + ret['retcode'] = 0 + mock_cmd = MagicMock(return_value=ret) + mock_exists = MagicMock(return_value=True) + + with patch.dict(zpool.__salt__, {'zpool.exists': mock_exists}): + with patch.dict(zpool.__salt__, {'cmd.run_all': mock_cmd}): + ret = zpool.scrub('mypool') + res = OrderedDict([('mypool', OrderedDict([('scrubbing', True)]))]) + self.assertEqual(res, ret) + + def test_scrub_pause(self): + ''' + Tests pause of scrub + ''' + ret = {} + ret['stdout'] = "" + ret['stderr'] = "" + ret['retcode'] = 0 + mock_cmd = MagicMock(return_value=ret) + mock_exists = MagicMock(return_value=True) + + with patch.dict(zpool.__salt__, {'zpool.exists': mock_exists}): + with patch.dict(zpool.__salt__, {'cmd.run_all': mock_cmd}): + ret = zpool.scrub('mypool', pause=True) + res = OrderedDict([('mypool', OrderedDict([('scrubbing', False)]))]) + self.assertEqual(res, ret) + + def test_scrub_stop(self): + ''' + Tests pauze of scrub + ''' + ret = {} + ret['stdout'] = "" + ret['stderr'] = "" + ret['retcode'] = 0 + mock_cmd = MagicMock(return_value=ret) + mock_exists = MagicMock(return_value=True) + + with patch.dict(zpool.__salt__, {'zpool.exists': mock_exists}): + with patch.dict(zpool.__salt__, {'cmd.run_all': mock_cmd}): + ret = zpool.scrub('mypool', stop=True) + res = OrderedDict([('mypool', OrderedDict([('scrubbing', False)]))]) + self.assertEqual(res, ret) + def test_split_success(self): ''' Tests split on success From 549f4806ceef2b43743f51a74599792d96986cde Mon Sep 17 00:00:00 2001 From: Roald Nefs Date: Sun, 19 Nov 2017 19:45:21 +0100 Subject: [PATCH 059/159] Fixed documentation in Mattermost module - Updated the code examples. - Added extra whitspace where needed. --- salt/modules/mattermost.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/salt/modules/mattermost.py b/salt/modules/mattermost.py index 58c77727cc0..7617d863b36 100644 --- a/salt/modules/mattermost.py +++ b/salt/modules/mattermost.py @@ -6,9 +6,10 @@ Module for sending messages to Mattermost :configuration: This module can be used by either passing an api_url and hook directly or by specifying both in a configuration profile in the salt - master/minion config. - For example: + master/minion config. For example: + .. code-block:: yaml + mattermost: hook: peWcBiMOS9HrZG15peWcBiMOS9HrZG15 api_url: https://example.com @@ -35,6 +36,7 @@ __virtualname__ = 'mattermost' def __virtual__(): ''' Return virtual name of the module. + :return: The virtual name of the module. ''' return __virtualname__ @@ -43,6 +45,7 @@ def __virtual__(): def _get_hook(): ''' Retrieves and return the Mattermost's configured hook + :return: String: the hook string ''' hook = __salt__['config.get']('mattermost.hook') or \ @@ -56,6 +59,7 @@ def _get_hook(): def _get_api_url(): ''' Retrieves and return the Mattermost's configured api url + :return: String: the api url string ''' api_url = __salt__['config.get']('mattermost.api_url') or \ @@ -69,6 +73,7 @@ def _get_api_url(): def _get_channel(): ''' Retrieves the Mattermost's configured channel + :return: String: the channel string ''' channel = __salt__['config.get']('mattermost.channel') or \ @@ -80,6 +85,7 @@ def _get_channel(): def _get_username(): ''' Retrieves the Mattermost's configured username + :return: String: the username string ''' username = __salt__['config.get']('mattermost.username') or \ @@ -95,14 +101,18 @@ def post_message(message, hook=None): ''' Send a message to a Mattermost channel. + :param channel: The channel name, either will work. :param username: The username of the poster. :param message: The message to send to the Mattermost channel. :param api_url: The Mattermost api url, if not specified in the configuration. :param hook: The Mattermost hook, if not specified in the configuration. :return: Boolean if message was sent successfully. + CLI Example: + .. code-block:: bash + salt '*' mattermost.post_message message='Build is done" ''' if not api_url: From 47147e2769209e4db8e968d29690292101307fde Mon Sep 17 00:00:00 2001 From: Roald Nefs Date: Mon, 20 Nov 2017 08:47:57 +0100 Subject: [PATCH 060/159] Added new Telegram module Added a new Telegram module, which allows to send a message via Telegram using the Telegram Bot API. The commit also includes: - Update for the Telegram returner (merged in #44148); the Telegram returner now fully utilizes the Telegram module. - Update to the documentation in order to add the Telegram module to the list list of all modules. - Updated Telegram returner test. - New Telegram module test. --- doc/ref/modules/all/index.rst | 1 + salt/modules/telegram.py | 143 +++++++++++++++++++ salt/returners/telegram_return.py | 62 +------- tests/unit/modules/test_telegram.py | 63 ++++++++ tests/unit/returners/test_telegram_return.py | 14 +- 5 files changed, 214 insertions(+), 69 deletions(-) create mode 100644 salt/modules/telegram.py create mode 100644 tests/unit/modules/test_telegram.py diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index 19e6aa9dc50..6ef587d1b42 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -417,6 +417,7 @@ execution modules system system_profiler systemd + telegram telemetry temp test diff --git a/salt/modules/telegram.py b/salt/modules/telegram.py new file mode 100644 index 00000000000..43f9df09701 --- /dev/null +++ b/salt/modules/telegram.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +''' +Module for sending messages via Telegram. + +:configuration: In order to send a message via the Telegram, certain + configuration is required in /etc/salt/minion on the relevant minions or + in the pillar. Some sample configs might look like:: + + telegram.chat_id: '123456789' + telegram.token: '00000000:xxxxxxxxxxxxxxxxxxxxxxxx' + +''' +from __future__ import absolute_import + +# Import Python libs +import logging + +from salt.exceptions import SaltInvocationError + +# Import 3rd-party libs +try: + import requests + HAS_REQUESTS = True +except ImportError: + HAS_REQUESTS = False + +log = logging.getLogger(__name__) + +__virtualname__ = 'telegram' + + +def __virtual__(): + ''' + Return virtual name of the module. + + :return: The virtual name of the module. + ''' + if not HAS_REQUESTS: + return False + return __virtualname__ + + +def _get_chat_id(): + ''' + Retrieves and return the Telegram's configured chat id + + :return: String: the chat id string + ''' + chat_id = __salt__['config.get']('telegram:chat_id') or \ + __salt__['config.get']('telegram.chat_id') + if not chat_id: + raise SaltInvocationError('No Telegram chat id found') + + return chat_id + + +def _get_token(): + ''' + Retrieves and return the Telegram's configured token + + :return: String: the token string + ''' + token = __salt__['config.get']('telegram:token') or \ + __salt__['config.get']('telegram.token') + if not token: + raise SaltInvocationError('No Telegram token found') + + return token + + +def post_message(message, chat_id=None, token=None): + ''' + Send a message to a Telegram chat. + + :param message: The message to send to the Telegram chat. + :param chat_id: (optional) The Telegram chat id. + :param token: (optional) The Telegram API token. + :return: Boolean if message was sent successfully. + + + CLI Example: + + .. code-block:: bash + + salt '*' telegram.post_message message="Hello Telegram!" + + ''' + if not chat_id: + chat_id = _get_chat_id() + + if not token: + token = _get_token() + + if not message: + log.error('message is a required option.') + + return _post_message(message=message, chat_id=chat_id, token=token) + + +def _post_message(message, chat_id, token): + ''' + Send a message to a Telegram chat. + + :param chat_id: The chat id. + :param message: The message to send to the telegram chat. + :param token: The Telegram API token. + :return: Boolean if message was sent successfully. + ''' + url = 'https://api.telegram.org/bot{0}/sendMessage'.format(token) + + parameters = dict() + if chat_id: + parameters['chat_id'] = chat_id + if message: + parameters['text'] = message + + try: + response = requests.post( + url, + data=parameters + ) + result = response.json() + + log.debug( + 'Raw response of the telegram request is {0}'.format(response) + ) + + except Exception: + log.exception( + 'Sending telegram api request failed' + ) + return False + + # Check if the Telegram Bot API returned successfully. + if not result.get('ok', False): + log.debug( + 'Sending telegram api request failed due to error {0} ({1})'.format( + result.get('error_code'), result.get('description') + ) + ) + return False + + return True diff --git a/salt/returners/telegram_return.py b/salt/returners/telegram_return.py index c818b276a81..ceacaa78a91 100644 --- a/salt/returners/telegram_return.py +++ b/salt/returners/telegram_return.py @@ -27,13 +27,6 @@ from __future__ import absolute_import # Import Python libs import logging -# Import 3rd-party libs -try: - import requests - HAS_REQUESTS = True -except ImportError: - HAS_REQUESTS = False - # Import Salt Libs import salt.returners @@ -48,8 +41,6 @@ def __virtual__(): :return: The virtual name of the module. ''' - if not HAS_REQUESTS: - return False return __virtualname__ @@ -61,7 +52,6 @@ def _get_options(ret=None): :return: Dictionary containing the data and options needed to send them to telegram. ''' - attrs = {'chat_id': 'chat_id', 'token': 'token'} @@ -81,7 +71,6 @@ def returner(ret): :param ret: The data to be sent. :return: Boolean if message was sent successfully. ''' - _options = _get_options(ret) chat_id = _options.get('chat_id') @@ -105,51 +94,6 @@ def returner(ret): ret.get('jid'), returns) - telegram = _post_message(chat_id, - message, - token) - - return telegram - - -def _post_message(chat_id, message, token): - ''' - Send a message to a Telegram chat. - - :param chat_id: The chat id. - :param message: The message to send to the telegram chat. - :param token: The Telegram API token. - :return: Boolean if message was sent successfully. - ''' - - url = 'https://api.telegram.org/bot{0}/sendMessage'.format(token) - - parameters = dict() - if chat_id: - parameters['chat_id'] = chat_id - if message: - parameters['text'] = message - - try: - response = requests.post( - url, - data=parameters - ) - result = response.json() - - log.debug( - 'Raw response of the telegram request is {0}'.format(response)) - - except Exception: - log.exception( - 'Sending telegram api request failed' - ) - result = False - - if response and 'message_id' in result: - success = True - else: - success = False - - log.debug('result {0}'.format(success)) - return bool(success) + return __salt__['telegram.post_message'](message, + chat_id=chat_id, + token=token) diff --git a/tests/unit/modules/test_telegram.py b/tests/unit/modules/test_telegram.py new file mode 100644 index 00000000000..c7e79d3006b --- /dev/null +++ b/tests/unit/modules/test_telegram.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +''' +Tests for the Telegram execution module. + +:codeauthor: :email:`Roald Nefs (info@roaldnefs.com)` +''' + +# Import Python Libs +from __future__ import absolute_import + +# Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import TestCase, skipIf +from tests.support.mock import ( + patch, + MagicMock, + NO_MOCK, + NO_MOCK_REASON +) + +# Import Salt Libs +import salt.modules.telegram as telegram + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class TelegramModuleTest(TestCase, LoaderModuleMockMixin): + ''' + Test cases for salt.modules.telegram. + ''' + def setup_loader_modules(self): + module_globals = { + telegram: { + '__salt__': { + 'config.get': MagicMock(return_value={ + 'telegram': { + 'chat_id': '123456789', + 'token': '000000000:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' + } + }) + } + } + } + if telegram.HAS_REQUESTS is False: + module_globals['sys.modules'] = {'requests': MagicMock()} + return module_globals + + def test_post_message(self): + ''' + Test the post_message function. + ''' + message = 'Hello World!' + + class MockRequests(object): + """ + Mock of requests response. + """ + def json(self): + return {'ok': True} + + with patch('salt.modules.telegram.requests.post', + MagicMock(return_value=MockRequests())): + + self.assertTrue(telegram.post_message(message)) diff --git a/tests/unit/returners/test_telegram_return.py b/tests/unit/returners/test_telegram_return.py index 7f63ab9b31e..e8a52465a65 100644 --- a/tests/unit/returners/test_telegram_return.py +++ b/tests/unit/returners/test_telegram_return.py @@ -38,16 +38,10 @@ class TelegramReturnerTestCase(TestCase, LoaderModuleMockMixin): options = {'chat_id': '', 'token': ''} - class MockRequest(object): - """ - Mock of requests response - """ - def json(self): - return {'message_id': ''} - with patch('salt.returners.telegram_return._get_options', MagicMock(return_value=options)), \ - patch('salt.returners.telegram_return.requests.post', - MagicMock(return_value=MockRequest())): - + patch.dict('salt.returners.telegram_return.__salt__', + {'telegram.post_message': MagicMock(return_value=True)} + ): self.assertTrue(telegram.returner(ret)) + From ac1a4d081775b699742b27b6577295d6fb330e87 Mon Sep 17 00:00:00 2001 From: Roald Nefs Date: Mon, 20 Nov 2017 10:41:08 +0100 Subject: [PATCH 061/159] Updated Telegram returner test - Removed trailing newlines --- tests/unit/returners/test_telegram_return.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/returners/test_telegram_return.py b/tests/unit/returners/test_telegram_return.py index e8a52465a65..417abcd8892 100644 --- a/tests/unit/returners/test_telegram_return.py +++ b/tests/unit/returners/test_telegram_return.py @@ -44,4 +44,3 @@ class TelegramReturnerTestCase(TestCase, LoaderModuleMockMixin): {'telegram.post_message': MagicMock(return_value=True)} ): self.assertTrue(telegram.returner(ret)) - From 90d1cb221d91a310951bed4968ec4f9d42b85532 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 17 Nov 2017 15:54:34 -0500 Subject: [PATCH 062/159] Add documentation about logging before modules are loaded --- doc/ref/modules/index.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/ref/modules/index.rst b/doc/ref/modules/index.rst index 3bd62b3c41f..cce2bde68a0 100644 --- a/doc/ref/modules/index.rst +++ b/doc/ref/modules/index.rst @@ -376,6 +376,22 @@ The above example will force the minion to use the :py:mod:`systemd .. __: https://github.com/saltstack/salt/issues/new +Logging Restrictions +-------------------- + +As a rule, logging should not be done anywhere in a Salt module before it is +loaded. This rule apples to all code that would run before the ``__virtual__()`` +function, as well as the code within the ``__virtual__()`` function itself. + +If logging statements are made before the virtual function determines if +the module should be loaded, then those logging statements will be called +repeatedly. This clutters up log files unnecessarily. + +Exceptions may be considered for logging statements made at the ``trace`` level. +However, it is better to provide the necessary information by another means. +One method is to :ref:`return error information ` in the +``__virtual__()`` function. + .. _modules-virtual-name: ``__virtualname__`` From f11d408389e2c65e20aab2f3ed98e4c2b45bae56 Mon Sep 17 00:00:00 2001 From: Ollie Armstrong Date: Mon, 20 Nov 2017 15:56:16 +0000 Subject: [PATCH 063/159] Fix acme state to correctly return on test Previously, the acme.cert state would return that a change was pending during a test run even in the case that the certificate would not have been touched. Changed the return value in this case so that it is not thought to require a change. This is reported in #40208 [0] and is also referenced in #42763 [1]. The issue #40208 looks to go on to recommend further changes beyond the scope of this 'quick fix'. [0] https://github.com/saltstack/salt/issues/40208#issuecomment-289637588 [1] https://github.com/saltstack/salt/issues/42763#issuecomment-345728031 --- salt/states/acme.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/states/acme.py b/salt/states/acme.py index 43649a64262..ad1c9d05644 100644 --- a/salt/states/acme.py +++ b/salt/states/acme.py @@ -85,6 +85,7 @@ def cert(name, comment += 'would have been renewed' else: comment += 'would not have been touched' + ret['result'] = True ret['comment'] = comment return ret From e2f11a0d2b915839834cb4bb49880b83ea6a8559 Mon Sep 17 00:00:00 2001 From: campbell Date: Mon, 20 Nov 2017 16:50:20 +0000 Subject: [PATCH 064/159] Add handling for FreeBSD in method zone_compare to avoid exception when /etc/localtime file does is absent. This is valid configuration on FreeBSD and represents UTC. --- salt/modules/timezone.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/salt/modules/timezone.py b/salt/modules/timezone.py index 1f10dbde43b..ff2bfb6394a 100644 --- a/salt/modules/timezone.py +++ b/salt/modules/timezone.py @@ -340,6 +340,10 @@ def zone_compare(timezone): if 'Solaris' in __grains__['os_family'] or 'AIX' in __grains__['os_family']: return timezone == get_zone() + if 'FreeBSD' in __grains__['os_family']: + if not os.path.isfile(_get_etc_localtime_path()): + return timezone == get_zone() + tzfile = _get_etc_localtime_path() zonepath = _get_zone_file(timezone) try: From f2a00ea3fc820ac57669dba3b756c5d2ea989bd5 Mon Sep 17 00:00:00 2001 From: Mike Place Date: Mon, 20 Nov 2017 11:16:43 -0700 Subject: [PATCH 065/159] Correct indentation --- salt/modules/timezone.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/timezone.py b/salt/modules/timezone.py index ff2bfb6394a..5b3376080c0 100644 --- a/salt/modules/timezone.py +++ b/salt/modules/timezone.py @@ -342,7 +342,7 @@ def zone_compare(timezone): if 'FreeBSD' in __grains__['os_family']: if not os.path.isfile(_get_etc_localtime_path()): - return timezone == get_zone() + return timezone == get_zone() tzfile = _get_etc_localtime_path() zonepath = _get_zone_file(timezone) From d26d9ff5e4c38760d51634b90d63196cc29135ab Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 20 Nov 2017 10:47:27 -0500 Subject: [PATCH 066/159] Handle timeout_monitor/TimeoutError issues for new versions of CherryPy Fixes #44601 --- salt/netapi/rest_cherrypy/app.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py index 06abfa9257e..17e52b98b1c 100644 --- a/salt/netapi/rest_cherrypy/app.py +++ b/salt/netapi/rest_cherrypy/app.py @@ -106,6 +106,13 @@ A REST API for Salt expire_responses : True Whether to check for and kill HTTP responses that have exceeded the default timeout. + + .. deprecated:: 2016.11.9, 2017.7.3, Oxygen + + The "expire_responses" configuration setting, which corresponds + to the ``timeout_monitor`` setting in CherryPy, is no longer + supported in CherryPy versions >= 12.0.0. + max_request_body_size : ``1048576`` Maximum size for the HTTP request body. collect_stats : False @@ -506,6 +513,7 @@ import salt.ext.six as six # Import Salt libs import salt import salt.auth +import salt.exceptions import salt.utils import salt.utils.event @@ -753,7 +761,8 @@ def hypermedia_handler(*args, **kwargs): except (salt.exceptions.SaltDaemonNotRunning, salt.exceptions.SaltReqTimeoutError) as exc: raise cherrypy.HTTPError(503, exc.strerror) - except (cherrypy.TimeoutError, salt.exceptions.SaltClientTimeout): + except (cherrypy.TimeoutError if hasattr(cherrypy, 'TimeoutError') else None, + salt.exceptions.SaltClientTimeout): raise cherrypy.HTTPError(504) except cherrypy.CherryPyException: raise @@ -2731,8 +2740,6 @@ class API(object): 'server.socket_port': self.apiopts.get('port', 8000), 'server.thread_pool': self.apiopts.get('thread_pool', 100), 'server.socket_queue_size': self.apiopts.get('queue_size', 30), - 'engine.timeout_monitor.on': self.apiopts.get( - 'expire_responses', True), 'max_request_body_size': self.apiopts.get( 'max_request_body_size', 1048576), 'debug': self.apiopts.get('debug', False), @@ -2750,6 +2757,14 @@ class API(object): }, } + if salt.utils.version_cmp(cherrypy.__version__, '12.0.0') < 0: + # CherryPy >= 12.0 no longer supports "timeout_monitor", only set + # this config option when using an older version of CherryPy. + # See Issue #44601 for more information. + conf['global']['engine.timeout_monitor.on'] = self.apiopts.get( + 'expire_responses', True + ) + if cpstats and self.apiopts.get('collect_stats', False): conf['/']['tools.cpstats.on'] = True From 477c3caa779376833416cdd9c1d8ecc1c3c3a95a Mon Sep 17 00:00:00 2001 From: rallytime Date: Mon, 20 Nov 2017 10:47:27 -0500 Subject: [PATCH 067/159] Handle timeout_monitor/TimeoutError issues for new versions of CherryPy Fixes #44601 --- salt/netapi/rest_cherrypy/app.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py index 7c5749ffa51..526f27ddc2f 100644 --- a/salt/netapi/rest_cherrypy/app.py +++ b/salt/netapi/rest_cherrypy/app.py @@ -102,6 +102,13 @@ A REST API for Salt expire_responses : True Whether to check for and kill HTTP responses that have exceeded the default timeout. + + .. deprecated:: 2016.11.9, 2017.7.3, Oxygen + + The "expire_responses" configuration setting, which corresponds + to the ``timeout_monitor`` setting in CherryPy, is no longer + supported in CherryPy versions >= 12.0.0. + max_request_body_size : ``1048576`` Maximum size for the HTTP request body. collect_stats : False @@ -606,8 +613,10 @@ import yaml # Import Salt libs import salt import salt.auth +import salt.exceptions import salt.utils.event import salt.utils.stringutils +import salt.utils.versions from salt.ext import six # Import salt-api libs @@ -854,7 +863,8 @@ def hypermedia_handler(*args, **kwargs): except (salt.exceptions.SaltDaemonNotRunning, salt.exceptions.SaltReqTimeoutError) as exc: raise cherrypy.HTTPError(503, exc.strerror) - except (cherrypy.TimeoutError, salt.exceptions.SaltClientTimeout): + except (cherrypy.TimeoutError if hasattr(cherrypy, 'TimeoutError') else None, + salt.exceptions.SaltClientTimeout): raise cherrypy.HTTPError(504) except cherrypy.CherryPyException: raise @@ -2839,8 +2849,6 @@ class API(object): 'server.socket_port': self.apiopts.get('port', 8000), 'server.thread_pool': self.apiopts.get('thread_pool', 100), 'server.socket_queue_size': self.apiopts.get('queue_size', 30), - 'engine.timeout_monitor.on': self.apiopts.get( - 'expire_responses', True), 'max_request_body_size': self.apiopts.get( 'max_request_body_size', 1048576), 'debug': self.apiopts.get('debug', False), @@ -2858,6 +2866,14 @@ class API(object): }, } + if salt.utils.versions.version_cmp(cherrypy.__version__, '12.0.0') < 0: + # CherryPy >= 12.0 no longer supports "timeout_monitor", only set + # this config option when using an older version of CherryPy. + # See Issue #44601 for more information. + conf['global']['engine.timeout_monitor.on'] = self.apiopts.get( + 'expire_responses', True + ) + if cpstats and self.apiopts.get('collect_stats', False): conf['/']['tools.cpstats.on'] = True From c9ba33432e811d78b1f72c63ee06933b311b3df2 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 20 Nov 2017 16:53:56 -0500 Subject: [PATCH 068/159] Add Non Base Environement salt:// source integration test --- .../files/file/prod/non-base-env.sls | 4 ++++ tests/integration/files/file/prod/nonbase_env | 1 + tests/integration/modules/test_state.py | 20 +++++++++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 tests/integration/files/file/prod/non-base-env.sls create mode 100644 tests/integration/files/file/prod/nonbase_env diff --git a/tests/integration/files/file/prod/non-base-env.sls b/tests/integration/files/file/prod/non-base-env.sls new file mode 100644 index 00000000000..63efe602937 --- /dev/null +++ b/tests/integration/files/file/prod/non-base-env.sls @@ -0,0 +1,4 @@ +test_file: + file.managed: + - name: /tmp/nonbase_env + - source: salt://nonbase_env diff --git a/tests/integration/files/file/prod/nonbase_env b/tests/integration/files/file/prod/nonbase_env new file mode 100644 index 00000000000..66012b6d044 --- /dev/null +++ b/tests/integration/files/file/prod/nonbase_env @@ -0,0 +1 @@ +it worked - new environment! diff --git a/tests/integration/modules/test_state.py b/tests/integration/modules/test_state.py index e75373f4154..3bca7972c3f 100644 --- a/tests/integration/modules/test_state.py +++ b/tests/integration/modules/test_state.py @@ -1240,3 +1240,23 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): self.assertIn(state_id, state_run) self.assertEqual(state_run[state_id]['comment'], 'Failure!') self.assertFalse(state_run[state_id]['result']) + + def test_state_nonbase_environment(self): + ''' + test state.sls with saltenv using a nonbase environment + with a salt source + ''' + state_run = self.run_function( + 'state.sls', + mods='non-base-env', + saltenv='prod' + ) + state_id = 'file_|-test_file_|-/tmp/nonbase_env_|-managed' + self.assertEqual(state_run[state_id]['comment'], 'File /tmp/nonbase_env updated') + self.assertTrue(state_run['file_|-test_file_|-/tmp/nonbase_env_|-managed']['result']) + self.assertTrue(os.path.isfile('/tmp/nonbase_env')) + + def tearDown(self): + nonbase_file = '/tmp/nonbase_env' + if os.path.isfile(nonbase_file): + os.remove(nonbase_file) From 74ededafa70c7ec5548d86289c6dbfc5e4cff6f2 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 20 Nov 2017 17:37:38 -0500 Subject: [PATCH 069/159] Add ssh thin_dir integration test --- tests/integration/ssh/test_deploy.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/integration/ssh/test_deploy.py b/tests/integration/ssh/test_deploy.py index cda96d8e494..1daab7ffabb 100644 --- a/tests/integration/ssh/test_deploy.py +++ b/tests/integration/ssh/test_deploy.py @@ -4,6 +4,8 @@ salt-ssh testing ''' # Import Python libs from __future__ import absolute_import +import os +import shutil # Import salt testing libs from tests.support.case import SSHCase @@ -19,3 +21,21 @@ class SSHTest(SSHCase): ''' ret = self.run_function('test.ping') self.assertTrue(ret, 'Ping did not return true') + + def test_thin_dir(self): + ''' + test to make sure thin_dir is created + and salt-call file is included + ''' + thin_dir = self.run_function('config.get', ['thin_dir'], wipe=False) + os.path.isdir(thin_dir) + os.path.exists(os.path.join(thin_dir, 'salt-call')) + os.path.exists(os.path.join(thin_dir, 'running_data')) + + def tearDown(self): + ''' + make sure to clean up any old ssh directories + ''' + salt_dir = self.run_function('config.get', ['thin_dir'], wipe=False) + if os.path.exists(salt_dir): + shutil.rmtree(salt_dir) From d0cf1468587284b0ce21e02f7292e0411a6d875a Mon Sep 17 00:00:00 2001 From: Volodymyr Samodid Date: Tue, 21 Nov 2017 14:29:09 +0200 Subject: [PATCH 070/159] fix for #44595 ** Overwrite output plugin for all client and runner instances in lxc salt-cloud driver --- salt/cloud/clouds/lxc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/cloud/clouds/lxc.py b/salt/cloud/clouds/lxc.py index bb26a8fab41..a343972fcb9 100644 --- a/salt/cloud/clouds/lxc.py +++ b/salt/cloud/clouds/lxc.py @@ -80,6 +80,7 @@ def _master_opts(cfg='master'): cfg = os.environ.get( 'SALT_MASTER_CONFIG', os.path.join(default_dir, cfg)) opts = config.master_config(cfg) + opts['output'] = 'quiet' return opts From 2cdf0f4dffa34ac3b12ca80f16f46f9cfe987f0f Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 21 Nov 2017 08:37:13 -0600 Subject: [PATCH 071/159] Fix traceback in new iSCSI IQN grains Using a simple str.find here means that any comments with InitiatorName in them would cause a traceback. This fixes the logic used to parse the IQN out of the file. This also removes a lot of needless creation of new lists just to use list.extend(), in favor of using list.append(). --- salt/grains/core.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index ce324ace8f6..affe3ef8560 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -2475,10 +2475,9 @@ def _linux_iqn(): if os.path.isfile(initiator): with salt.utils.files.fopen(initiator, 'r') as _iscsi: for line in _iscsi: - if line.find('InitiatorName') != -1: - iqn = line.split('=') - final_iqn = iqn[1].rstrip() - ret.extend([final_iqn]) + line = line.strip() + if line.startswith('InitiatorName='): + ret.append(line.split('=', 1)[1]) return ret @@ -2492,9 +2491,10 @@ def _aix_iqn(): aixret = __salt__['cmd.run'](aixcmd) if aixret[0].isalpha(): - iqn = aixret.split() - final_iqn = iqn[1].rstrip() - ret.extend([final_iqn]) + try: + ret.append(aixret.split()[1].rstrip()) + except IndexError: + pass return ret @@ -2507,8 +2507,7 @@ def _linux_wwns(): for fcfile in glob.glob('/sys/class/fc_host/*/port_name'): with salt.utils.files.fopen(fcfile, 'r') as _wwn: for line in _wwn: - line = line.rstrip() - ret.extend([line[2:]]) + ret.append(line.rstrip()[2:]) return ret @@ -2534,8 +2533,7 @@ def _windows_iqn(): for line in cmdret['stdout'].splitlines(): if line[0].isalpha(): continue - line = line.rstrip() - ret.extend([line]) + ret.append(line.rstrip()) return ret @@ -2551,7 +2549,6 @@ def _windows_wwns(): cmdret = __salt__['cmd.run_ps'](ps_cmd) for line in cmdret: - line = line.rstrip() - ret.append(line) + ret.append(line.rstrip()) return ret From 669d03ec33b59bc29688ce74e69faaf0caee668a Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 21 Nov 2017 10:21:47 -0500 Subject: [PATCH 072/159] Move TimoutError check lower down in exception list When called as a one-liner, the "else None" causes the following error: TypeError: catching classes that do not inherit from BaseException is not allowed --- salt/netapi/rest_cherrypy/app.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py index 526f27ddc2f..19938188980 100644 --- a/salt/netapi/rest_cherrypy/app.py +++ b/salt/netapi/rest_cherrypy/app.py @@ -863,12 +863,18 @@ def hypermedia_handler(*args, **kwargs): except (salt.exceptions.SaltDaemonNotRunning, salt.exceptions.SaltReqTimeoutError) as exc: raise cherrypy.HTTPError(503, exc.strerror) - except (cherrypy.TimeoutError if hasattr(cherrypy, 'TimeoutError') else None, - salt.exceptions.SaltClientTimeout): + except salt.exceptions.SaltClientTimeout: raise cherrypy.HTTPError(504) except cherrypy.CherryPyException: raise except Exception as exc: + # The TimeoutError exception class was removed in CherryPy in 12.0.0, but + # Still check existence of TimeoutError and handle in CherryPy < 12. + # The check was moved down from the SaltClientTimeout error line because + # A one-line if statement throws a BaseException inheritance TypeError. + if hasattr(cherrypy, 'TimeoutError') and isinstance(exc, cherrypy.TimeoutError): + raise cherrypy.HTTPError(504) + import traceback logger.debug("Error while processing request for: %s", From 359a59dd64a4d7823b9f8ba9393ca2bc0e7ada23 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Tue, 21 Nov 2017 10:40:25 -0500 Subject: [PATCH 073/159] Add salt-key -d integration test --- tests/integration/shell/test_key.py | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/integration/shell/test_key.py b/tests/integration/shell/test_key.py index fa6d5c36091..dbd5fb366d7 100644 --- a/tests/integration/shell/test_key.py +++ b/tests/integration/shell/test_key.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import os import shutil import tempfile +import textwrap # Import Salt Testing libs from tests.support.case import ShellCase @@ -56,6 +57,36 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin): if USERA in user: self.run_call('user.delete {0} remove=True'.format(USERA)) + def test_remove_key(self): + ''' + test salt-key -d usage + ''' + min_name = 'minibar' + pki_dir = self.master_opts['pki_dir'] + key = os.path.join(pki_dir, 'minions', min_name) + + with salt.utils.fopen(key, 'w') as fp: + fp.write(textwrap.dedent('''\ + -----BEGIN PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoqIZDtcQtqUNs0wC7qQz + JwFhXAVNT5C8M8zhI+pFtF/63KoN5k1WwAqP2j3LquTG68WpxcBwLtKfd7FVA/Kr + OF3kXDWFnDi+HDchW2lJObgfzLckWNRFaF8SBvFM2dys3CGSgCV0S/qxnRAjrJQb + B3uQwtZ64ncJAlkYpArv3GwsfRJ5UUQnYPDEJwGzMskZ0pHd60WwM1gMlfYmNX5O + RBEjybyNpYDzpda6e6Ypsn6ePGLkP/tuwUf+q9wpbRE3ZwqERC2XRPux+HX2rGP+ + mkzpmuHkyi2wV33A9pDfMgRHdln2CLX0KgfRGixUQhW1o+Kmfv2rq4sGwpCgLbTh + NwIDAQAB + -----END PUBLIC KEY----- + ''')) + + check_key = self.run_key('-p {0}'.format(min_name)) + self.assertIn('Accepted Keys:', check_key) + self.assertIn('minibar: -----BEGIN PUBLIC KEY-----', check_key) + + remove_key = self.run_key('-d {0} -y'.format(min_name)) + + check_key = self.run_key('-p {0}'.format(min_name)) + self.assertEqual([], check_key) + def test_list_accepted_args(self): ''' test salt-key -l for accepted arguments From 2fb9f3e0f707c0f4bb15db43569a39bb386e2b20 Mon Sep 17 00:00:00 2001 From: Daniel Wallace Date: Sun, 12 Nov 2017 15:09:38 -0700 Subject: [PATCH 074/159] ignore_retcode in lsattr Before using it a lot and spamming error messages --- salt/modules/file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index f7145d06775..6fe3303460d 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -552,11 +552,11 @@ def lsattr(path): raise SaltInvocationError("File or directory does not exist.") cmd = ['lsattr', path] - result = __salt__['cmd.run'](cmd, python_shell=False) + result = __salt__['cmd.run'](cmd, ignore_retcode=True, python_shell=False) results = {} for line in result.splitlines(): - if not line.startswith('lsattr'): + if not line.startswith('lsattr: '): vals = line.split(None, 1) results[vals[1]] = re.findall(r"[acdijstuADST]", vals[0]) From 628f015c1bc30a9f0c50afc67a85dc13f0382c38 Mon Sep 17 00:00:00 2001 From: rallytime Date: Tue, 21 Nov 2017 10:21:47 -0500 Subject: [PATCH 075/159] Move TimoutError check lower down in exception list When called as a one-liner, the "else None" causes the following error: TypeError: catching classes that do not inherit from BaseException is not allowed --- salt/netapi/rest_cherrypy/app.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py index 17e52b98b1c..becb45b0a55 100644 --- a/salt/netapi/rest_cherrypy/app.py +++ b/salt/netapi/rest_cherrypy/app.py @@ -761,12 +761,18 @@ def hypermedia_handler(*args, **kwargs): except (salt.exceptions.SaltDaemonNotRunning, salt.exceptions.SaltReqTimeoutError) as exc: raise cherrypy.HTTPError(503, exc.strerror) - except (cherrypy.TimeoutError if hasattr(cherrypy, 'TimeoutError') else None, - salt.exceptions.SaltClientTimeout): + except salt.exceptions.SaltClientTimeout: raise cherrypy.HTTPError(504) except cherrypy.CherryPyException: raise except Exception as exc: + # The TimeoutError exception class was removed in CherryPy in 12.0.0, but + # Still check existence of TimeoutError and handle in CherryPy < 12. + # The check was moved down from the SaltClientTimeout error line because + # A one-line if statement throws a BaseException inheritance TypeError. + if hasattr(cherrypy, 'TimeoutError') and isinstance(exc, cherrypy.TimeoutError): + raise cherrypy.HTTPError(504) + import traceback logger.debug("Error while processing request for: %s", From 98f2170bec20be171bc8f465bbba019348faa1f0 Mon Sep 17 00:00:00 2001 From: Christian McHugh Date: Wed, 22 Nov 2017 14:52:07 +0000 Subject: [PATCH 076/159] add is_installed function --- salt/modules/pip.py | 55 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/salt/modules/pip.py b/salt/modules/pip.py index 21dd43d0e17..6e9d1f2a90d 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -1153,6 +1153,61 @@ def list_upgrades(bin_env=None, return packages +def is_installed(pkgname=None, + bin_env=None, + user=None, + cwd=None): + ''' + Filter list of installed apps from ``freeze`` and return True or False if + ``pkgname`` exists in the list of packages installed. + + .. note:: + + If the version of pip available is older than 8.0.3, the packages + wheel, setuptools, and distribute will not be reported by this function + even if they are installed. Unlike + :py:func:`pip.freeze `, this function always + reports the version of pip which is installed. + + CLI Example: + + .. code-block:: bash + + salt '*' pip.is_installed salt + + .. versionadded:: Oxygen + + The packages wheel, setuptools, and distribute are included if the + installed pip is new enough. + ''' + for line in freeze(bin_env=bin_env, user=user, cwd=cwd): + if line.startswith('-f') or line.startswith('#'): + # ignore -f line as it contains --find-links directory + # ignore comment lines + continue + elif line.startswith('-e hg+not trust'): + # ignore hg + not trust problem + continue + elif line.startswith('-e'): + line = line.split('-e ')[1] + version_, name = line.split('#egg=') + elif len(line.split('===')) >= 2: + name = line.split('===')[0] + version_ = line.split('===')[1] + elif len(line.split('==')) >= 2: + name = line.split('==')[0] + version_ = line.split('==')[1] + else: + logger.error('Can\'t parse line \'{0}\''.format(line)) + continue + + if pkgname: + if pkgname == name.lower(): + return True + + return False + + def upgrade_available(pkg, bin_env=None, user=None, From 1dea504f2d74ea6740f8572cb3abe2679bcb2307 Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 22 Nov 2017 12:44:36 -0500 Subject: [PATCH 077/159] Replace util path references with new ones --- salt/daemons/masterapi.py | 3 ++- salt/modules/firewalld.py | 13 +++++++++---- salt/states/firewalld.py | 9 ++++++--- salt/utils/verify.py | 2 +- tests/integration/shell/test_key.py | 2 +- tests/integration/states/test_service.py | 2 +- tests/unit/states/test_archive.py | 10 +++++----- 7 files changed, 25 insertions(+), 16 deletions(-) diff --git a/salt/daemons/masterapi.py b/salt/daemons/masterapi.py index 4219688ad3c..3bf8262d6e5 100644 --- a/salt/daemons/masterapi.py +++ b/salt/daemons/masterapi.py @@ -42,6 +42,7 @@ import salt.utils.platform import salt.utils.stringutils import salt.utils.user import salt.utils.verify +import salt.utils.versions from salt.defaults import DEFAULT_TARGET_DELIM from salt.pillar import git_pillar from salt.exceptions import FileserverConfigError, SaltMasterError @@ -534,7 +535,7 @@ class RemoteFuncs(object): return ret expr_form = load.get('expr_form') if expr_form is not None and 'tgt_type' not in load: - salt.utils.warn_until( + salt.utils.versions.warn_until( u'Neon', u'_mine_get: minion {0} uses pre-Nitrogen API key ' u'"expr_form". Accepting for backwards compatibility ' diff --git a/salt/modules/firewalld.py b/salt/modules/firewalld.py index e21a6eebf68..400a744ece9 100644 --- a/salt/modules/firewalld.py +++ b/salt/modules/firewalld.py @@ -13,6 +13,7 @@ import re # Import Salt Libs from salt.exceptions import CommandExecutionError import salt.utils.path +import salt.utils.versions log = logging.getLogger(__name__) @@ -635,8 +636,10 @@ def add_port(zone, port, permanent=True, force_masquerade=None): # This will be deprecated in a future release if force_masquerade is None: force_masquerade = True - salt.utils.warn_until('Neon', - 'add_port function will no longer force enable masquerading in future releases. Use add_masquerade to enable masquerading.') + salt.utils.versions.warn_until( + 'Neon', + 'add_port function will no longer force enable masquerading ' + 'in future releases. Use add_masquerade to enable masquerading.') # (DEPRECATED) Force enable masquerading # TODO: remove in future release @@ -709,8 +712,10 @@ def add_port_fwd(zone, src, dest, proto='tcp', dstaddr='', permanent=True, force # This will be deprecated in a future release if force_masquerade is None: force_masquerade = True - salt.utils.warn_until('Neon', - 'add_port_fwd function will no longer force enable masquerading in future releases. Use add_masquerade to enable masquerading.') + salt.utils.versions.warn_until( + 'Neon', + 'add_port_fwd function will no longer force enable masquerading ' + 'in future releases. Use add_masquerade to enable masquerading.') # (DEPRECATED) Force enable masquerading # TODO: remove in future release diff --git a/salt/states/firewalld.py b/salt/states/firewalld.py index a93798ba7b0..fe1477982e4 100644 --- a/salt/states/firewalld.py +++ b/salt/states/firewalld.py @@ -82,8 +82,9 @@ import logging # Import Salt Libs from salt.exceptions import CommandExecutionError -import salt.utils.path from salt.output import nested +import salt.utils.path +import salt.utils.versions log = logging.getLogger(__name__) @@ -231,8 +232,10 @@ def present(name, # if prune_services == None, set to True and log a deprecation warning if prune_services is None: prune_services = True - salt.utils.warn_until('Neon', - 'The \'prune_services\' argument default is currently True, but will be changed to True in future releases.') + salt.utils.versions.warn_until( + 'Neon', + 'The \'prune_services\' argument default is currently True, ' + 'but will be changed to True in future releases.') ret = _present(name, block_icmp, prune_block_icmp, default, masquerade, ports, prune_ports, port_fwd, prune_port_fwd, services, prune_services, interfaces, prune_interfaces, diff --git a/salt/utils/verify.py b/salt/utils/verify.py index 70ac9db4fe0..f1d2dda6b7f 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -34,7 +34,7 @@ import salt.utils.versions log = logging.getLogger(__name__) -ROOT_DIR = 'c:\\salt' if salt.utils.is_windows() else '/' +ROOT_DIR = 'c:\\salt' if salt.utils.platform.is_windows() else '/' def zmq_version(): diff --git a/tests/integration/shell/test_key.py b/tests/integration/shell/test_key.py index 3c0ba348b8c..6ae22b3d391 100644 --- a/tests/integration/shell/test_key.py +++ b/tests/integration/shell/test_key.py @@ -66,7 +66,7 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin): pki_dir = self.master_opts['pki_dir'] key = os.path.join(pki_dir, 'minions', min_name) - with salt.utils.fopen(key, 'w') as fp: + with salt.utils.files.fopen(key, 'w') as fp: fp.write(textwrap.dedent('''\ -----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoqIZDtcQtqUNs0wC7qQz diff --git a/tests/integration/states/test_service.py b/tests/integration/states/test_service.py index b9c8e150146..72689372e58 100644 --- a/tests/integration/states/test_service.py +++ b/tests/integration/states/test_service.py @@ -31,7 +31,7 @@ class ServiceTest(ModuleCase, SaltReturnAssertsMixin): self.service_name = 'systemd-journald' cmd_name = 'systemctl' - if salt.utils.which(cmd_name) is None: + if salt.utils.path.which(cmd_name) is None: self.skipTest('{0} is not installed'.format(cmd_name)) def check_service_status(self, exp_return): diff --git a/tests/unit/states/test_archive.py b/tests/unit/states/test_archive.py index f1d3999a59a..1c23e921a00 100644 --- a/tests/unit/states/test_archive.py +++ b/tests/unit/states/test_archive.py @@ -20,7 +20,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.states.archive as archive from salt.ext.six.moves import zip # pylint: disable=import-error,redefined-builtin -import salt.utils +import salt.utils.platform def _isfile_side_effect(path): @@ -63,7 +63,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): archive.extracted tar options ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): source = 'c:\\tmp\\foo.tar.gz' tmp_dir = 'c:\\tmp\\test_extracted_tar' else: @@ -112,7 +112,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'file.source_list': mock_source_list}),\ patch.dict(archive.__states__, {'file.directory': mock_true}),\ patch.object(os.path, 'isfile', isfile_mock),\ - patch('salt.utils.which', MagicMock(return_value=True)): + patch('salt.utils.path.which', MagicMock(return_value=True)): for test_opts, ret_opts in zip(test_tar_opts, ret_tar_opts): archive.extracted(tmp_dir, source, options=test_opts, @@ -152,7 +152,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'file.source_list': mock_source_list}),\ patch.dict(archive.__states__, {'file.directory': mock_true}),\ patch.object(os.path, 'isfile', isfile_mock),\ - patch('salt.utils.which', MagicMock(return_value=True)): + patch('salt.utils.path.which', MagicMock(return_value=True)): ret = archive.extracted(os.path.join(os.sep + 'tmp', 'out'), source, options='xvzf', @@ -191,7 +191,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'file.source_list': mock_source_list}),\ patch.dict(archive.__states__, {'file.directory': mock_true}),\ patch.object(os.path, 'isfile', isfile_mock),\ - patch('salt.utils.which', MagicMock(return_value=True)): + patch('salt.utils.path.which', MagicMock(return_value=True)): ret = archive.extracted(os.path.join(os.sep + 'tmp', 'out'), source, options='xvzf', From 1c309c7c4e41bc483ea1f659661979086f064f26 Mon Sep 17 00:00:00 2001 From: skjaro Date: Wed, 22 Nov 2017 22:06:40 +0100 Subject: [PATCH 078/159] Fix state absent broken by ipset-add-new-options PR --- salt/states/ipset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/states/ipset.py b/salt/states/ipset.py index 194fa5172cb..fe41d25c8ec 100644 --- a/salt/states/ipset.py +++ b/salt/states/ipset.py @@ -293,7 +293,7 @@ def absent(name, entry=None, entries=None, family='ipv4', **kwargs): kwargs['set_name'], family) else: - command = __salt__['ipset.delete'](kwargs['set_name'], _entry, family, **kwargs) + command = __salt__['ipset.delete'](kwargs['set_name'], entry, family, **kwargs) if 'Error' not in command: ret['changes'] = {'locale': name} ret['result'] = True From 4356d022b9c40195eb80938f8c28dcf6d4131c99 Mon Sep 17 00:00:00 2001 From: Ollie Armstrong Date: Thu, 23 Nov 2017 11:11:42 +0000 Subject: [PATCH 079/159] Fix acme.cert to run certbot non-interactively Certbot should never be running interactively when executed via Salt, so this patch adds an argument to certbot which informs it to not run interactively. This fixes #42763. --- salt/modules/acme.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/salt/modules/acme.py b/salt/modules/acme.py index af0ab0175f0..931806c1e43 100644 --- a/salt/modules/acme.py +++ b/salt/modules/acme.py @@ -125,8 +125,7 @@ def cert(name, salt 'gitlab.example.com' acme.cert dev.example.com "[gitlab.example.com]" test_cert=True renew=14 webroot=/opt/gitlab/embedded/service/gitlab-rails/public ''' - # cmd = [LEA, 'certonly', '--quiet'] - cmd = [LEA, 'certonly'] + cmd = [LEA, 'certonly', '--non-interactive'] cert_file = _cert_file(name, 'cert') if not __salt__['file.file_exists'](cert_file): From ab5520fdf7cb019c3bc8d97bac5883a9a4ba2a78 Mon Sep 17 00:00:00 2001 From: Christian McHugh Date: Thu, 23 Nov 2017 14:08:46 +0000 Subject: [PATCH 080/159] add true and false test for is_installed --- tests/unit/modules/test_pip.py | 56 ++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/unit/modules/test_pip.py b/tests/unit/modules/test_pip.py index 7d7735ff708..fed4f2dd45a 100644 --- a/tests/unit/modules/test_pip.py +++ b/tests/unit/modules/test_pip.py @@ -1109,6 +1109,62 @@ class PipTestCase(TestCase, LoaderModuleMockMixin): } ) + def test_is_installed_true(self): + eggs = [ + 'M2Crypto==0.21.1', + '-e git+git@github.com:s0undt3ch/salt-testing.git@9ed81aa2f918d59d3706e56b18f0782d1ea43bf8#egg=SaltTesting-dev', + 'bbfreeze==1.1.0', + 'bbfreeze-loader==1.1.0', + 'pycrypto==2.6' + ] + mock = MagicMock( + return_value={ + 'retcode': 0, + 'stdout': '\n'.join(eggs) + } + ) + with patch.dict(pip.__salt__, {'cmd.run_all': mock}): + with patch('salt.modules.pip.version', + MagicMock(return_value='6.1.1')): + ret = pip.is_installed(pkgname='bbfreeze') + mock.assert_called_with( + ['pip', 'freeze'], + cwd=None, + runas=None, + python_shell=False, + use_vt=False, + ) + self.assertTrue(ret) + + + def test_is_installed_false(self): + eggs = [ + 'M2Crypto==0.21.1', + '-e git+git@github.com:s0undt3ch/salt-testing.git@9ed81aa2f918d59d3706e56b18f0782d1ea43bf8#egg=SaltTesting-dev', + 'bbfreeze==1.1.0', + 'bbfreeze-loader==1.1.0', + 'pycrypto==2.6' + ] + mock = MagicMock( + return_value={ + 'retcode': 0, + 'stdout': '\n'.join(eggs) + } + ) + with patch.dict(pip.__salt__, {'cmd.run_all': mock}): + with patch('salt.modules.pip.version', + MagicMock(return_value='6.1.1')): + ret = pip.is_installed(pkgname='notexist') + mock.assert_called_with( + ['pip', 'freeze'], + cwd=None, + runas=None, + python_shell=False, + use_vt=False, + ) + self.assertFalse(ret) + + def test_install_pre_argument_in_resulting_command(self): pkg = 'pep8' # Lower than 1.4 versions don't end-up with `--pre` in the resulting From 1a9971065a24bd2562aca47b6a0e459525f74b43 Mon Sep 17 00:00:00 2001 From: Arthur Lutz Date: Fri, 24 Nov 2017 14:07:17 +0100 Subject: [PATCH 081/159] typo in "def disable", s/enable/disable/ in ret['comment'] --- salt/modules/schedule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/schedule.py b/salt/modules/schedule.py index da186888530..487713c5ca1 100644 --- a/salt/modules/schedule.py +++ b/salt/modules/schedule.py @@ -771,7 +771,7 @@ def disable(**kwargs): return ret except KeyError: # Effectively a no-op, since we can't really return without an event system - ret['comment'] = 'Event module not available. Schedule enable job failed.' + ret['comment'] = 'Event module not available. Schedule disable job failed.' return ret From 9e45e4eff9405faa7d33d61ebb6e6a436270a30e Mon Sep 17 00:00:00 2001 From: Nicole Thomas Date: Mon, 27 Nov 2017 11:25:05 -0500 Subject: [PATCH 082/159] Lint: Remove extra blank lines --- tests/unit/modules/test_pip.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit/modules/test_pip.py b/tests/unit/modules/test_pip.py index fed4f2dd45a..f11b0c757d6 100644 --- a/tests/unit/modules/test_pip.py +++ b/tests/unit/modules/test_pip.py @@ -1136,7 +1136,6 @@ class PipTestCase(TestCase, LoaderModuleMockMixin): ) self.assertTrue(ret) - def test_is_installed_false(self): eggs = [ 'M2Crypto==0.21.1', @@ -1164,7 +1163,6 @@ class PipTestCase(TestCase, LoaderModuleMockMixin): ) self.assertFalse(ret) - def test_install_pre_argument_in_resulting_command(self): pkg = 'pep8' # Lower than 1.4 versions don't end-up with `--pre` in the resulting From 94001ffc64548c6ebe1556023fa4b3930cac6d39 Mon Sep 17 00:00:00 2001 From: Nicole Thomas Date: Mon, 27 Nov 2017 16:15:44 -0500 Subject: [PATCH 083/159] lint: whitespace fixes --- tests/unit/grains/test_core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py index 8f1a5f4d0a3..1285db4c117 100644 --- a/tests/unit/grains/test_core.py +++ b/tests/unit/grains/test_core.py @@ -512,7 +512,7 @@ PATCHLEVEL = 3 self.assertEqual(_grains.get('iscsi_iqn'), ['iqn.1993-08.org.debian:01:d12f7aba36']) - + @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') def test_linux_memdata(self): ''' @@ -666,4 +666,4 @@ SwapTotal: 4789244 kB''' os_grains = core.os_data() self.assertEqual(os_grains.get('mem_total'), 2023) - self.assertEqual(os_grains.get('swap_total'), 400) \ No newline at end of file + self.assertEqual(os_grains.get('swap_total'), 400) From c729eef9e87d905610272befbb54725671d0f122 Mon Sep 17 00:00:00 2001 From: Eric Radman Date: Mon, 27 Nov 2017 18:17:35 -0500 Subject: [PATCH 084/159] Fix invocation of cron.file with when run with test=True --- salt/states/cron.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/states/cron.py b/salt/states/cron.py index 584407c0678..0d202b82b5a 100644 --- a/salt/states/cron.py +++ b/salt/states/cron.py @@ -392,9 +392,9 @@ def absent(name, The special keyword used in the job (eg. @reboot, @hourly...). Quotes must be used, otherwise PyYAML will strip the '@' sign. ''' - ### NOTE: The keyword arguments in **kwargs are ignored in this state, but - ### cannot be removed from the function definition, otherwise the use - ### of unsupported arguments will result in a traceback. + # NOTE: The keyword arguments in **kwargs are ignored in this state, but + # cannot be removed from the function definition, otherwise the use + # of unsupported arguments will result in a traceback. name = name.strip() if identifier is False: @@ -566,6 +566,7 @@ def file(name, user, group, mode, + [], # no special attrs for cron template, context, defaults, From 1f893d1e952d1369f66522278582e5acd9da908d Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 27 Nov 2017 18:50:10 -0600 Subject: [PATCH 085/159] Fix ssh state.show_top integration test 63823f3 changed how we handle cases where master_tops and top file matches both exist in a given environment. This updates the test to reflect the correct behavior in develop based on the fact that the master_tops_first config option is set to False. See: https://github.com/saltstack/salt/pull/44697#issuecomment-347373412 and the subsequent comments in that pull request. --- tests/integration/ssh/test_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/ssh/test_state.py b/tests/integration/ssh/test_state.py index 26343b4efed..88ce7a660b5 100644 --- a/tests/integration/ssh/test_state.py +++ b/tests/integration/ssh/test_state.py @@ -57,7 +57,7 @@ class SSHStateTest(SSHCase): test state.show_top with salt-ssh ''' ret = self.run_function('state.show_top') - self.assertEqual(ret, {u'base': [u'master_tops_test', u'core']}) + self.assertEqual(ret, {u'base': [u'core', u'master_tops_test']}) def test_state_single(self): ''' From 871a90cbc567933952aaf61f1a33ab64fb76cc94 Mon Sep 17 00:00:00 2001 From: Zach Moody Date: Fri, 3 Nov 2017 22:39:56 -0500 Subject: [PATCH 086/159] add netbox execution module --- salt/modules/netbox.py | 126 ++++++++++++++++++++++++++++++ tests/unit/modules/test_netbox.py | 82 +++++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 salt/modules/netbox.py create mode 100644 tests/unit/modules/test_netbox.py diff --git a/salt/modules/netbox.py b/salt/modules/netbox.py new file mode 100644 index 00000000000..a40ed19d7b8 --- /dev/null +++ b/salt/modules/netbox.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +''' +NetBox +====== + +Module to query NetBox + +:codeauthor: Zach Moody +:maturity: new +:depends: pynetbox + +The following config should be in the minion config file. In order to +work with ``secrets`` you should provide a token and path to your +private key file: + +.. code-block:: yaml + + netbox: + url: + token: + keyfile: + +.. versionadded:: Oxygen +''' + +from __future__ import absolute_import +import logging + +from salt.exceptions import CommandExecutionError +from salt.utils.args import clean_kwargs + +log = logging.getLogger(__name__) + +try: + import pynetbox + HAS_PYNETBOX = True +except ImportError: + HAS_PYNETBOX = False + +AUTH_ENDPOINTS = ( + 'secrets', +) + + +def __virtual__(): + ''' + pynetbox must be installed. + ''' + if not HAS_PYNETBOX: + return ( + False, + 'The netbox execution module cannot be loaded: ' + 'pynetbox library is not installed.' + ) + else: + return True + + +def _config(): + config = __salt__['config.get']('netbox') + if not config: + raise CommandExecutionError( + 'NetBox execution module configuration could not be found' + ) + return config + + +def _nb_obj(auth_required=False): + pynb_kwargs = {} + if auth_required: + pynb_kwargs['token'] = _config().get('token') + pynb_kwargs['private_key_file'] = _config().get('keyfile') + return pynetbox.api(_config().get('url'), **pynb_kwargs) + + +def _strip_url_field(input_dict): + if 'url' in input_dict.keys(): + del input_dict['url'] + for k, v in input_dict.items(): + if isinstance(v, dict): + _strip_url_field(v) + return input_dict + + +def filter(app, endpoint, **kwargs): + ''' + Get a list of items from NetBox. + + .. code-block:: bash + + salt myminion netbox.filter dcim devices status=1 role=router + ''' + ret = [] + nb = _nb_obj(auth_required=True if app in AUTH_ENDPOINTS else False) + nb_query = getattr(getattr(nb, app), endpoint).filter( + **clean_kwargs(**kwargs) + ) + if nb_query: + ret = [_strip_url_field(dict(i)) for i in nb_query] + return sorted(ret) + + +def get(app, endpoint, id=None, **kwargs): + ''' + Get a single item from NetBox. + + To get an item based on ID. + + .. code-block:: bash + + salt myminion netbox.get dcim devices id=123 + + Or using named arguments that correspond with accepted filters on + the NetBox endpoint. + + .. code-block:: bash + + salt myminion netbox.get dcim devices name=my-router + ''' + nb = _nb_obj(auth_required=True if app in AUTH_ENDPOINTS else False) + if id: + return dict(getattr(getattr(nb, app), endpoint).get(id)) + else: + return dict( + getattr(getattr(nb, app), endpoint).get(**clean_kwargs(**kwargs)) + ) diff --git a/tests/unit/modules/test_netbox.py b/tests/unit/modules/test_netbox.py new file mode 100644 index 00000000000..4f2722fcfc2 --- /dev/null +++ b/tests/unit/modules/test_netbox.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Zach Moody ` +''' + +# Import Python Libs +from __future__ import absolute_import + +try: + import pynetbox + HAS_PYNETBOX = True +except ImportError: + HAS_PYNETBOX = False + +# Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import TestCase, skipIf +from tests.support.mock import ( + patch, + MagicMock, + call, + NO_MOCK, + NO_MOCK_REASON +) +import salt.modules.netbox as netbox + +NETBOX_RESPONSE_STUB = { + 'device_name': 'test1-router1', + 'url': 'http://test/', + 'device_role': { + 'name': 'router', + 'url': 'http://test/' + } +} + + +@skipIf(HAS_PYNETBOX is False, 'pynetbox lib not installed') +@skipIf(NO_MOCK, NO_MOCK_REASON) +@patch('salt.modules.netbox._config', MagicMock()) +class NetBoxTestCase(TestCase, LoaderModuleMockMixin): + + def setup_loader_modules(self): + return { + netbox: {}, + } + + def test_get_by_id(self): + with patch('pynetbox.api', MagicMock()) as mock: + netbox.get('dcim', 'devices', id=1) + self.assertEqual( + mock.mock_calls[1], + call().dcim.devices.get(1) + ) + + def test_get_by_name(self): + with patch('pynetbox.api', MagicMock()) as mock: + netbox.get('dcim', 'devices', name='test') + self.assertEqual( + mock.mock_calls[1], + call().dcim.devices.get(name='test') + ) + + def test_filter_by_site(self): + with patch('pynetbox.api', MagicMock()) as mock: + netbox.filter('dcim', 'devices', site='test') + self.assertEqual( + mock.mock_calls[1], + call().dcim.devices.filter(site='test') + ) + + def test_filter_url(self): + strip_url = netbox._strip_url_field(NETBOX_RESPONSE_STUB) + self.assertTrue( + 'url' not in strip_url and 'url' not in strip_url['device_role'] + ) + + def test_get_secret(self): + with patch('pynetbox.api', MagicMock()) as mock: + netbox.get('secrets', 'secrets', name='test') + self.assertTrue( + 'token' and 'private_key_file' in mock.call_args[1] + ) From 1faa685d9e26ad61f19986fec991e69f7f0c15d2 Mon Sep 17 00:00:00 2001 From: Zach Moody Date: Mon, 27 Nov 2017 19:05:51 -0600 Subject: [PATCH 087/159] fixup pylint import error. --- tests/unit/modules/test_netbox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/modules/test_netbox.py b/tests/unit/modules/test_netbox.py index 4f2722fcfc2..c316d0618aa 100644 --- a/tests/unit/modules/test_netbox.py +++ b/tests/unit/modules/test_netbox.py @@ -7,7 +7,7 @@ from __future__ import absolute_import try: - import pynetbox + import pynetbox # pylint: disable=unused-import HAS_PYNETBOX = True except ImportError: HAS_PYNETBOX = False From 682fc1b884cb5d477be4224338edfcccd6ae5dd4 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Mon, 27 Nov 2017 21:37:46 -0600 Subject: [PATCH 088/159] Fix failing grains test on CentOS Resolves https://github.com/saltstack/salt-jenkins/issues/556 --- tests/unit/grains/test_core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py index a2772305e5a..073cb3ad471 100644 --- a/tests/unit/grains/test_core.py +++ b/tests/unit/grains/test_core.py @@ -477,7 +477,8 @@ PATCHLEVEL = 3 '/proc/meminfo': True } _cmd_run_map = { - 'dpkg --print-architecture': 'amd64' + 'dpkg --print-architecture': 'amd64', + 'rpm --eval %{_host_cpu}': 'x86_64' } path_exists_mock = MagicMock(side_effect=lambda x: _path_exists_map[x]) From ba1424ec6ffe2adab721f141b7a8e8176d83de9e Mon Sep 17 00:00:00 2001 From: Volodymyr Samodid Date: Tue, 28 Nov 2017 08:12:59 +0200 Subject: [PATCH 089/159] fix #44651 **now salt-cloud lxc driver won't exit if provider target unavalible --- salt/cloud/clouds/lxc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/salt/cloud/clouds/lxc.py b/salt/cloud/clouds/lxc.py index a343972fcb9..64d0238fa50 100644 --- a/salt/cloud/clouds/lxc.py +++ b/salt/cloud/clouds/lxc.py @@ -560,9 +560,10 @@ def get_configured_provider(vm_=None): # in all cases, verify that the linked saltmaster is alive. if data: ret = _salt('test.ping', salt_target=data['target']) - if not ret: - raise SaltCloudSystemExit( + if ret: + return data + else: + log.error( 'Configured provider {0} minion: {1} is unreachable'.format( __active_provider_name__, data['target'])) - return data return False From 678da78a61f4b1f8452861eddfec78d7947029c2 Mon Sep 17 00:00:00 2001 From: spenceation Date: Tue, 28 Nov 2017 13:46:44 -0500 Subject: [PATCH 090/159] Added new functions: - get_dos_blocks - get_local_admins - test_fib_route - test_security_policy --- salt/modules/panos.py | 156 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) diff --git a/salt/modules/panos.py b/salt/modules/panos.py index 1fd0438ef16..597991f2f29 100644 --- a/salt/modules/panos.py +++ b/salt/modules/panos.py @@ -519,6 +519,22 @@ def get_domain_config(): return __proxy__['panos.call'](query) +def get_dos_blocks(): + ''' + Show the DoS block-ip table. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_dos_blocks + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + def get_fqdn_cache(): ''' Print FQDNs used in rules and their IPs. @@ -846,6 +862,32 @@ def get_lldp_neighbors(): return __proxy__['panos.call'](query) +def get_local_admins(): + ''' + Show all local administrator accounts. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_local_admins + + ''' + admin_list = get_users_config() + response = [] + + if 'users' not in admin_list['result']: + return response + + if isinstance(admin_list['result']['users']['entry'], list): + for entry in admin_list['result']['users']['entry']: + response.append(entry['name']) + else: + response.append(admin_list['result']['users']['entry']['name']) + + return response + + def get_logdb_quota(): ''' Report the logdb quotas. @@ -2157,6 +2199,120 @@ def shutdown(): return __proxy__['panos.call'](query) +def test_fib_route(ip=None, + vr='vr1'): + ''' + Perform a route lookup within active route table (fib). + + ip (str): The destination IP address to test. + + vr (str): The name of the virtual router to test. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.test_fib_route 4.2.2.2 + salt '*' panos.test_fib_route 4.2.2.2 my-vr + + ''' + + xpath = "" + + if ip: + xpath += "{0}".format(ip) + + if vr: + xpath += "{0}".format(vr) + + xpath += "" + + query = {'type': 'op', + 'cmd': xpath} + + return __proxy__['panos.call'](query) + + +def test_security_policy(sourcezone=None, + destinationzone=None, + source=None, + destination=None, + protocol=None, + port=None, + application=None, + category=None, + vsys='1', + allrules=False): + ''' + Checks which security policy as connection will match on the device. + + sourcezone (str): The source zone matched against the connection. + + destinationzone (str): The destination zone matched against the connection. + + source (str): The source address. This must be a single IP address. + + destination (str): The destination address. This must be a single IP address. + + protocol (int): The protocol number for the connection. This is the numerical representation of the protocol. + + port (int): The port number for the connection. + + application (str): The application that should be matched. + + category (str): The category that should be matched. + + vsys (int): The numerical representation of the VSYS ID. + + allrules (bool): Show all potential match rules until first allow rule. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.test_security_policy sourcezone=trust destinationzone=untrust protocol=6 port=22 + salt '*' panos.test_security_policy sourcezone=trust destinationzone=untrust protocol=6 port=22 vsys=2 + + ''' + + xpath = "" + + if sourcezone: + xpath += "{0}".format(sourcezone) + + if destinationzone: + xpath += "{0}".format(destinationzone) + + if source: + xpath += "{0}".format(source) + + if destination: + xpath += "{0}".format(destination) + + if protocol: + xpath += "{0}".format(protocol) + + if port: + xpath += "{0}".format(port) + + if application: + xpath += "{0}".format(application) + + if category: + xpath += "{0}".format(category) + + if allrules: + xpath += "yes" + + xpath += "" + + query = {'type': 'op', + 'vsys': "vsys{0}".format(vsys), + 'cmd': xpath} + + return __proxy__['panos.call'](query) + + def unlock_admin(username=None): ''' Unlocks a locked administrator account. From 8446138b4c5cd1ee064829e16d5533a36a9cf999 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 28 Nov 2017 16:47:54 -0700 Subject: [PATCH 091/159] Add prototype of master stats --- salt/master.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/salt/master.py b/salt/master.py index 6b8bec3d5a2..1dd779ab1e6 100644 --- a/salt/master.py +++ b/salt/master.py @@ -16,6 +16,7 @@ import errno import signal import stat import logging +import collections import multiprocessing import salt.serializers.msgpack @@ -797,6 +798,7 @@ class MWorker(salt.utils.process.SignalHandlingMultiprocessingProcess): :return: Master worker ''' kwargs[u'name'] = name + self.name = name super(MWorker, self).__init__(**kwargs) self.opts = opts self.req_channels = req_channels @@ -804,6 +806,8 @@ class MWorker(salt.utils.process.SignalHandlingMultiprocessingProcess): self.mkey = mkey self.key = key self.k_mtime = 0 + self.stats = collections.defaultdict(lambda: {'mean': 0, 'runs': 0}) + self.stat_clock = time.time() # We need __setstate__ and __getstate__ to also pickle 'SMaster.secrets'. # Otherwise, 'SMaster.secrets' won't be copied over to the spawned process @@ -903,10 +907,24 @@ class MWorker(salt.utils.process.SignalHandlingMultiprocessingProcess): if u'cmd' not in data: log.error(u'Received malformed command %s', data) return {} + cmd = data[u'cmd'] log.trace(u'AES payload received with command %s', data[u'cmd']) - if data[u'cmd'].startswith(u'__'): + if cmd.startswith(u'__'): return False - return self.aes_funcs.run_func(data[u'cmd'], data) + if self.opts[u'master_stats']: + start = time.time() + self.stats[cmd][u'runs'] += 1 + ret = self.aes_funcs.run_func(data[u'cmd'], data) + if self.opts[u'master_stats']: + end = time.time() + duration = end - start + self.stats[cmd][u'mean'] = (self.stats[cmd][u'mean'] * (self.stats[cmd][u'runs'] - 1) + duration) / self.stats[cmd][u'runs'] + if end - self.stat_clock < self.opts[u'master_stats_event_time']: + # Fire the event with the stats and wipe the tracker + self.event.fire_event({u'time': end - self.stat_clock, u'worker': self.name, u'stats': self.stats}, tagify(u'stats', u'refresh', u'minion')) + self.stats = collections.defaultdict(lambda: {'mean': 0, 'runs': 0}) + self.stat_clock = end + return ret def run(self): ''' From 4bde80b90747ba0b79bb2ebe7d0ef1d7edc57996 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Tue, 28 Nov 2017 16:49:55 -0700 Subject: [PATCH 092/159] fix ref to the event bus --- salt/master.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/master.py b/salt/master.py index 1dd779ab1e6..f250461c8f4 100644 --- a/salt/master.py +++ b/salt/master.py @@ -921,7 +921,7 @@ class MWorker(salt.utils.process.SignalHandlingMultiprocessingProcess): self.stats[cmd][u'mean'] = (self.stats[cmd][u'mean'] * (self.stats[cmd][u'runs'] - 1) + duration) / self.stats[cmd][u'runs'] if end - self.stat_clock < self.opts[u'master_stats_event_time']: # Fire the event with the stats and wipe the tracker - self.event.fire_event({u'time': end - self.stat_clock, u'worker': self.name, u'stats': self.stats}, tagify(u'stats', u'refresh', u'minion')) + self.aes_funcs.event.fire_event({u'time': end - self.stat_clock, u'worker': self.name, u'stats': self.stats}, tagify(u'stats', u'refresh', u'minion')) self.stats = collections.defaultdict(lambda: {'mean': 0, 'runs': 0}) self.stat_clock = end return ret From 73bae346cd08b8bb2c510aa948630766697296ac Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 28 Nov 2017 18:39:12 -0800 Subject: [PATCH 093/159] Swapping logic in _windows_iqn back to previous method to ensure tests pass. --- salt/grains/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index ccc7c5be1de..da541edf56c 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -2531,9 +2531,9 @@ def _windows_iqn(): wmic, namespace, mspath, get)) for line in cmdret['stdout'].splitlines(): - if line[0].isalpha(): - continue - ret.append(line.rstrip()) + if line.startswith('iqn.'): + line = line.rstrip() + ret.append(line.rstrip()) return ret From 5e8eff9ff16e4b3ac68df439a27e6612e3c10cf1 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 28 Nov 2017 19:42:16 -0600 Subject: [PATCH 094/159] Add __context__ to runner/wheel loader instances This allows us to use __context__ to set a retcode, which the client can use to set the 'success' value. --- salt/client/mixins.py | 6 +++++- salt/loader.py | 11 +++++++--- salt/runner.py | 7 +++++-- salt/states/saltmod.py | 47 +++++++++++++++++++----------------------- salt/wheel/__init__.py | 3 ++- 5 files changed, 41 insertions(+), 33 deletions(-) diff --git a/salt/client/mixins.py b/salt/client/mixins.py index 76a202fcdd3..383a6b228e6 100644 --- a/salt/client/mixins.py +++ b/salt/client/mixins.py @@ -385,7 +385,11 @@ class SyncClientMixin(object): # Initialize a context for executing the method. with tornado.stack_context.StackContext(self.functions.context_dict.clone): data[u'return'] = self.functions[fun](*args, **kwargs) - data[u'success'] = True + try: + data[u'success'] = self.context.get(u'retcode', 0) == 0 + except AttributeError: + # Assume a True result if no context attribute + data[u'success'] = True if isinstance(data[u'return'], dict) and u'data' in data[u'return']: # some functions can return boolean values data[u'success'] = salt.utils.state.check_result(data[u'return'][u'data']) diff --git a/salt/loader.py b/salt/loader.py index 81e44ebb27a..14d9fe4ac65 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -372,15 +372,18 @@ def tops(opts): return FilterDictWrapper(ret, u'.top') -def wheels(opts, whitelist=None): +def wheels(opts, whitelist=None, context=None): ''' Returns the wheels modules ''' + if context is None: + context = {} return LazyLoader( _module_dirs(opts, u'wheel'), opts, tag=u'wheel', whitelist=whitelist, + pack={u'__context__': context}, ) @@ -836,17 +839,19 @@ def call(fun, **kwargs): return funcs[fun](*args) -def runner(opts, utils=None): +def runner(opts, utils=None, context=None): ''' Directly call a function inside a loader directory ''' if utils is None: utils = {} + if context is None: + context = {} ret = LazyLoader( _module_dirs(opts, u'runners', u'runner', ext_type_dirs=u'runner_dirs'), opts, tag=u'runners', - pack={u'__utils__': utils}, + pack={u'__utils__': utils, u'__context__': context}, ) # TODO: change from __salt__ to something else, we overload __salt__ too much ret.pack[u'__salt__'] = ret diff --git a/salt/runner.py b/salt/runner.py index 988ce9609b7..c31d524edd4 100644 --- a/salt/runner.py +++ b/salt/runner.py @@ -43,6 +43,7 @@ class RunnerClient(mixins.SyncClientMixin, mixins.AsyncClientMixin, object): def __init__(self, opts): self.opts = opts + self.context = {} @property def functions(self): @@ -51,11 +52,13 @@ class RunnerClient(mixins.SyncClientMixin, mixins.AsyncClientMixin, object): self.utils = salt.loader.utils(self.opts) # Must be self.functions for mixin to work correctly :-/ try: - self._functions = salt.loader.runner(self.opts, utils=self.utils) + self._functions = salt.loader.runner( + self.opts, utils=self.utils, context=self.context) except AttributeError: # Just in case self.utils is still not present (perhaps due to # problems with the loader), load the runner funcs without them - self._functions = salt.loader.runner(self.opts) + self._functions = salt.loader.runner( + self.opts, context=self.context) return self._functions diff --git a/salt/states/saltmod.py b/salt/states/saltmod.py index 256662b84f2..4752388555c 100644 --- a/salt/states/saltmod.py +++ b/salt/states/saltmod.py @@ -787,28 +787,16 @@ def runner(name, **kwargs): runner_return = out.get('return') if isinstance(runner_return, dict) and 'Error' in runner_return: out['success'] = False - if not out.get('success', True): - cmt = "Runner function '{0}' failed{1}.".format( + + success = out.get('success', True) + ret = {'name': name, + 'changes': {}, + 'result': success} + ret['comment'] = "Runner function '{0}' {1}{2}.".format( name, + 'executed' if success else 'failed', ' with return {0}'.format(runner_return) if runner_return else '', ) - ret = { - 'name': name, - 'result': False, - 'changes': {}, - 'comment': cmt, - } - else: - cmt = "Runner function '{0}' executed{1}.".format( - name, - ' with return {0}'.format(runner_return) if runner_return else '', - ) - ret = { - 'name': name, - 'result': True, - 'changes': {}, - 'comment': cmt, - } ret['__orchestration__'] = True if 'jid' in out: @@ -1039,15 +1027,22 @@ def wheel(name, **kwargs): __env__=__env__, **kwargs) - ret['result'] = True + wheel_return = out.get('return') + if isinstance(wheel_return, dict) and 'Error' in wheel_return: + out['success'] = False + + success = out.get('success', True) + ret = {'name': name, + 'changes': {}, + 'result': success} + ret['comment'] = "Wheel function '{0}' {1}{2}.".format( + name, + 'executed' if success else 'failed', + ' with return {0}'.format(wheel_return) if wheel_return else '', + ) + ret['__orchestration__'] = True if 'jid' in out: ret['__jid__'] = out['jid'] - runner_return = out.get('return') - ret['comment'] = "Wheel function '{0}' executed{1}.".format( - name, - ' with return {0}'.format(runner_return) if runner_return else '', - ) - return ret diff --git a/salt/wheel/__init__.py b/salt/wheel/__init__.py index af462befb88..ad0d4c14782 100644 --- a/salt/wheel/__init__.py +++ b/salt/wheel/__init__.py @@ -43,7 +43,8 @@ class WheelClient(salt.client.mixins.SyncClientMixin, def __init__(self, opts=None): self.opts = opts - self.functions = salt.loader.wheels(opts) + self.context = {} + self.functions = salt.loader.wheels(opts, context=self.context) # TODO: remove/deprecate def call_func(self, fun, **kwargs): From 18905cf32be8aff2362f8a8364426ab45484db83 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Tue, 28 Nov 2017 21:34:19 -0600 Subject: [PATCH 095/159] Add integration test for triggering orchestration failure via context --- .../file/base/_runners/runtests_helpers.py | 16 ++++++++++ .../file/base/_wheel/runtests_helpers.py | 16 ++++++++++ .../files/file/base/orch/retcode.sls | 15 ++++++++++ tests/integration/runners/test_state.py | 29 +++++++++++++++++++ 4 files changed, 76 insertions(+) create mode 100644 tests/integration/files/file/base/_runners/runtests_helpers.py create mode 100644 tests/integration/files/file/base/_wheel/runtests_helpers.py create mode 100644 tests/integration/files/file/base/orch/retcode.sls diff --git a/tests/integration/files/file/base/_runners/runtests_helpers.py b/tests/integration/files/file/base/_runners/runtests_helpers.py new file mode 100644 index 00000000000..df816a92a54 --- /dev/null +++ b/tests/integration/files/file/base/_runners/runtests_helpers.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +''' +Runner functions for integration tests +''' + +# Import python libs +from __future__ import absolute_import + + +def failure(): + __context__['retcode'] = 1 + return False + + +def success(): + return True diff --git a/tests/integration/files/file/base/_wheel/runtests_helpers.py b/tests/integration/files/file/base/_wheel/runtests_helpers.py new file mode 100644 index 00000000000..136d2c0a28c --- /dev/null +++ b/tests/integration/files/file/base/_wheel/runtests_helpers.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +''' +Wheel functions for integration tests +''' + +# Import python libs +from __future__ import absolute_import + + +def failure(): + __context__['retcode'] = 1 + return False + + +def success(): + return True diff --git a/tests/integration/files/file/base/orch/retcode.sls b/tests/integration/files/file/base/orch/retcode.sls new file mode 100644 index 00000000000..d46c6e70875 --- /dev/null +++ b/tests/integration/files/file/base/orch/retcode.sls @@ -0,0 +1,15 @@ +test_runner_success: + salt.runner: + - name: runtests_helpers.success + +test_runner_failure: + salt.runner: + - name: runtests_helpers.failure + +test_wheel_success: + salt.wheel: + - name: runtests_helpers.success + +test_wheel_failure: + salt.wheel: + - name: runtests_helpers.failure diff --git a/tests/integration/runners/test_state.py b/tests/integration/runners/test_state.py index a774476817b..4d1e543c0fe 100644 --- a/tests/integration/runners/test_state.py +++ b/tests/integration/runners/test_state.py @@ -106,6 +106,35 @@ class StateRunnerTest(ShellCase): for item in out: self.assertIn(item, ret) + def test_orchestrate_retcode(self): + ''' + Test orchestration with nonzero retcode set in __context__ + ''' + self.run_run('saltutil.sync_runners') + self.run_run('saltutil.sync_wheel') + ret = '\n'.join(self.run_run('state.orchestrate orch.retcode')) + + for result in (' ID: test_runner_success\n' + ' Function: salt.runner\n' + ' Name: runtests_helpers.success\n' + ' Result: True', + + ' ID: test_runner_failure\n' + ' Function: salt.runner\n' + ' Name: runtests_helpers.failure\n' + ' Result: False', + + ' ID: test_wheel_success\n' + ' Function: salt.wheel\n' + ' Name: runtests_helpers.success\n' + ' Result: True', + + ' ID: test_wheel_failure\n' + ' Function: salt.wheel\n' + ' Name: runtests_helpers.failure\n' + ' Result: False'): + self.assertIn(result, ret) + def test_orchestrate_target_doesnt_exists(self): ''' test orchestration when target doesnt exist From 443f58f2a16335812d99cbd9d3723cd0080276c8 Mon Sep 17 00:00:00 2001 From: spenceation Date: Wed, 29 Nov 2017 08:33:23 -0500 Subject: [PATCH 096/159] Fixed LINT issues. --- salt/modules/panos.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/panos.py b/salt/modules/panos.py index 597991f2f29..ed9ef5fa8a4 100644 --- a/salt/modules/panos.py +++ b/salt/modules/panos.py @@ -875,16 +875,16 @@ def get_local_admins(): ''' admin_list = get_users_config() response = [] - + if 'users' not in admin_list['result']: return response - + if isinstance(admin_list['result']['users']['entry'], list): for entry in admin_list['result']['users']['entry']: response.append(entry['name']) else: response.append(admin_list['result']['users']['entry']['name']) - + return response From ecdd2732316ada0672df1beefb7580fc22cca673 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 29 Nov 2017 09:47:56 -0600 Subject: [PATCH 097/159] Add info about runner/wheel retcodes to Oxygen release notes --- doc/topics/releases/oxygen.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/doc/topics/releases/oxygen.rst b/doc/topics/releases/oxygen.rst index 188ef506eba..6a18804b72b 100644 --- a/doc/topics/releases/oxygen.rst +++ b/doc/topics/releases/oxygen.rst @@ -25,6 +25,25 @@ by any master tops matches that are not matched via a top file. To make master tops matches execute first, followed by top file matches, set the new :conf_minion:`master_tops_first` minion config option to ``True``. +Return Codes for Runner/Wheel Functions +--------------------------------------- + +When using :ref:`orchestration `, runner and wheel +functions used to report a ``True`` result if the function ran to completion +without raising an exception. It is now possible to set a return code in the +``__context__`` dictionary, allowing runner and wheel functions to report that +they failed. Here's some example pseudocode: + +.. code-block:: python + + def myrunner(): + ... + do stuff + ... + if some_error_condition: + __context__['retcode'] = 1 + return result + LDAP via External Authentication Changes ---------------------------------------- In this release of Salt, if LDAP Bind Credentials are supplied, then From 0e37ad39c3282c11ace18879f76f84917d0fafce Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 29 Nov 2017 10:20:22 -0600 Subject: [PATCH 098/159] Add retcode documentation to orchestrate docs --- doc/ref/states/writing.rst | 2 ++ doc/topics/orchestrate/orchestrate_runner.rst | 32 +++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/doc/ref/states/writing.rst b/doc/ref/states/writing.rst index 5e94c1ccc34..78b181ddf7f 100644 --- a/doc/ref/states/writing.rst +++ b/doc/ref/states/writing.rst @@ -111,6 +111,8 @@ This code will call the `managed` function in the :mod:`file ` state module and pass the arguments ``name`` and ``source`` to it. +.. _state-return-data: + Return Data =========== diff --git a/doc/topics/orchestrate/orchestrate_runner.rst b/doc/topics/orchestrate/orchestrate_runner.rst index 55dd253e064..82d8facde9f 100644 --- a/doc/topics/orchestrate/orchestrate_runner.rst +++ b/doc/topics/orchestrate/orchestrate_runner.rst @@ -5,10 +5,10 @@ Orchestrate Runner ================== Executing states or highstate on a minion is perfect when you want to ensure that -minion configured and running the way you want. Sometimes however you want to +minion configured and running the way you want. Sometimes however you want to configure a set of minions all at once. -For example, if you want to set up a load balancer in front of a cluster of web +For example, if you want to set up a load balancer in front of a cluster of web servers you can ensure the load balancer is set up first, and then the same matching configuration is applied consistently across the whole cluster. @@ -222,6 +222,34 @@ To execute with pillar data. "master": "mymaster"}' +Return Codes in Runner/Wheel Jobs +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: Oxygen + +State (``salt.state``) jobs are able to report failure via the :ref:`state +return dictionary `. Remote execution (``salt.function``) +jobs are able to report failure by setting a ``retcode`` key in the +``__context__`` dictionary. However, runner (``salt.runner``) and wheel +(``salt.wheel``) jobs would only report a ``False`` result when the +runner/wheel function raised an exception. As of the Oxygen release, it is now +possible to set a retcode in runner and wheel functions just as you can do in +remote execution functions. Here is some example pseudocode: + +.. code-block:: python + + def myrunner(): + ... + do stuff + ... + if some_error_condition: + __context__['retcode'] = 1 + return result + +This allows a custom runner/wheel function to report its failure so that +requisites can accurately tell that a job has failed. + + More Complex Orchestration ~~~~~~~~~~~~~~~~~~~~~~~~~~ From 943a524e77bce6eb95686e044df1abca4f6e9223 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 29 Nov 2017 10:31:06 -0600 Subject: [PATCH 099/159] Add wheel/runner return to changes dict for orchestration --- salt/states/saltmod.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/salt/states/saltmod.py b/salt/states/saltmod.py index 4752388555c..0470c7f8708 100644 --- a/salt/states/saltmod.py +++ b/salt/states/saltmod.py @@ -790,13 +790,12 @@ def runner(name, **kwargs): success = out.get('success', True) ret = {'name': name, - 'changes': {}, + 'changes': {'return': runner_return}, 'result': success} - ret['comment'] = "Runner function '{0}' {1}{2}.".format( - name, - 'executed' if success else 'failed', - ' with return {0}'.format(runner_return) if runner_return else '', - ) + ret['comment'] = "Runner function '{0}' {1}.".format( + name, + 'executed' if success else 'failed', + ) ret['__orchestration__'] = True if 'jid' in out: @@ -1033,13 +1032,12 @@ def wheel(name, **kwargs): success = out.get('success', True) ret = {'name': name, - 'changes': {}, + 'changes': {'return': wheel_return}, 'result': success} - ret['comment'] = "Wheel function '{0}' {1}{2}.".format( - name, - 'executed' if success else 'failed', - ' with return {0}'.format(wheel_return) if wheel_return else '', - ) + ret['comment'] = "Wheel function '{0}' {1}.".format( + name, + 'executed' if success else 'failed', + ) ret['__orchestration__'] = True if 'jid' in out: From cf97cd8034488ca130c9262ed588ab45ad98b67a Mon Sep 17 00:00:00 2001 From: rallytime Date: Wed, 29 Nov 2017 12:24:47 -0500 Subject: [PATCH 100/159] Bump deprecation notice in utils/args from Oxygen to Fluorine --- salt/utils/args.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/utils/args.py b/salt/utils/args.py index b5fcc6f5889..51db5d23624 100644 --- a/salt/utils/args.py +++ b/salt/utils/args.py @@ -451,10 +451,10 @@ def format_call(fun, continue extra[key] = copy.deepcopy(value) - # We'll be showing errors to the users until Salt Oxygen comes out, after + # We'll be showing errors to the users until Salt Fluorine comes out, after # which, errors will be raised instead. salt.utils.versions.warn_until( - 'Oxygen', + 'Fluorine', 'It\'s time to start raising `SaltInvocationError` instead of ' 'returning warnings', # Let's not show the deprecation warning on the console, there's no @@ -491,7 +491,7 @@ def format_call(fun, '{0}. If you were trying to pass additional data to be used ' 'in a template context, please populate \'context\' with ' '\'key: value\' pairs. Your approach will work until Salt ' - 'Oxygen is out.{1}'.format( + 'Fluorine is out.{1}'.format( msg, '' if 'full' not in ret else ' Please update your state files.' ) From 74a18f387ff3a1161dcf60fb6b9a5083159e0b5c Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 29 Nov 2017 15:51:53 -0600 Subject: [PATCH 101/159] Update orchestration unit tests --- tests/unit/states/test_saltmod.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/states/test_saltmod.py b/tests/unit/states/test_saltmod.py index a884e331d08..4929a34c41d 100644 --- a/tests/unit/states/test_saltmod.py +++ b/tests/unit/states/test_saltmod.py @@ -258,8 +258,8 @@ class SaltmodTestCase(TestCase, LoaderModuleMockMixin): ''' name = 'state' - ret = {'changes': {}, 'name': 'state', 'result': True, - 'comment': 'Runner function \'state\' executed with return True.', + ret = {'changes': {'return': True}, 'name': 'state', 'result': True, + 'comment': 'Runner function \'state\' executed.', '__orchestration__': True} runner_mock = MagicMock(return_value={'return': True}) @@ -274,8 +274,8 @@ class SaltmodTestCase(TestCase, LoaderModuleMockMixin): ''' name = 'state' - ret = {'changes': {}, 'name': 'state', 'result': True, - 'comment': 'Wheel function \'state\' executed with return True.', + ret = {'changes': {'return': True}, 'name': 'state', 'result': True, + 'comment': 'Wheel function \'state\' executed.', '__orchestration__': True} wheel_mock = MagicMock(return_value={'return': True}) From 3ef6beeb6681773c1791e1ffde6de2e2981f7506 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 29 Nov 2017 17:11:23 -0700 Subject: [PATCH 102/159] clean up and add basic config --- salt/config/__init__.py | 6 ++++++ salt/master.py | 33 +++++++++++++++++++++++---------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 5680f727b94..7b0a8b2ce4a 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -165,6 +165,10 @@ VALID_OPTS = { # The master_pubkey_signature must also be set for this. 'master_use_pubkey_signature': bool, + # Enable master stats eveents to be fired, these events will contain information about + # what commands the master is processing and what the rates are of the executions + 'master_stats': bool, + 'master_stats_event_iter': int # The key fingerprint of the higher-level master for the syndic to verify it is talking to the # intended master 'syndic_finger': str, @@ -1515,6 +1519,8 @@ DEFAULT_MASTER_OPTS = { 'svnfs_saltenv_whitelist': [], 'svnfs_saltenv_blacklist': [], 'max_event_size': 1048576, + 'master_stats': False, + 'master_stats_event_iter': 60, 'minionfs_env': 'base', 'minionfs_mountpoint': '', 'minionfs_whitelist': [], diff --git a/salt/master.py b/salt/master.py index f250461c8f4..aac98cf0acf 100644 --- a/salt/master.py +++ b/salt/master.py @@ -883,6 +883,19 @@ class MWorker(salt.utils.process.SignalHandlingMultiprocessingProcess): u'clear': self._handle_clear}[key](load) raise tornado.gen.Return(ret) + def _post_stats(self, start, cmd): + ''' + Calculate the master stats and fire events with stat info + ''' + end = time.time() + duration = end - start + self.stats[cmd][u'mean'] = (self.stats[cmd][u'mean'] * (self.stats[cmd][u'runs'] - 1) + duration) / self.stats[cmd][u'runs'] + if end - self.stat_clock < self.opts[u'master_stats_event_iter']: + # Fire the event with the stats and wipe the tracker + self.aes_funcs.event.fire_event({u'time': end - self.stat_clock, u'worker': self.name, u'stats': self.stats}, tagify(u'stats', u'refresh', u'minion')) + self.stats = collections.defaultdict(lambda: {'mean': 0, 'runs': 0}) + self.stat_clock = end + def _handle_clear(self, load): ''' Process a cleartext command @@ -892,9 +905,16 @@ class MWorker(salt.utils.process.SignalHandlingMultiprocessingProcess): the command specified in the load's 'cmd' key. ''' log.trace(u'Clear payload received with command %s', load[u'cmd']) - if load[u'cmd'].startswith(u'__'): + cmd = load[u'cmd'] + if cmd.startswith(u'__'): return False - return getattr(self.clear_funcs, load[u'cmd'])(load), {u'fun': u'send_clear'} + if self.opts[u'master_stats']: + start = time.time() + self.stats[cmd][u'runs'] += 1 + ret = getattr(self.clear_funcs, cmd)(load), {u'fun': u'send_clear'} + if self.opts[u'master_stats']: + self._post_stats(start, cmd) + return ret def _handle_aes(self, data): ''' @@ -916,14 +936,7 @@ class MWorker(salt.utils.process.SignalHandlingMultiprocessingProcess): self.stats[cmd][u'runs'] += 1 ret = self.aes_funcs.run_func(data[u'cmd'], data) if self.opts[u'master_stats']: - end = time.time() - duration = end - start - self.stats[cmd][u'mean'] = (self.stats[cmd][u'mean'] * (self.stats[cmd][u'runs'] - 1) + duration) / self.stats[cmd][u'runs'] - if end - self.stat_clock < self.opts[u'master_stats_event_time']: - # Fire the event with the stats and wipe the tracker - self.aes_funcs.event.fire_event({u'time': end - self.stat_clock, u'worker': self.name, u'stats': self.stats}, tagify(u'stats', u'refresh', u'minion')) - self.stats = collections.defaultdict(lambda: {'mean': 0, 'runs': 0}) - self.stat_clock = end + self._post_stats(start, cmd) return ret def run(self): From 4a808827c898dfa50ef58d9425149d8ad638d16a Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Wed, 29 Nov 2017 17:14:17 -0700 Subject: [PATCH 103/159] Add a default config entry to the master config --- conf/master | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/master b/conf/master index d1f1d1099bf..5426ddff371 100644 --- a/conf/master +++ b/conf/master @@ -297,6 +297,11 @@ #batch_safe_limit: 100 #batch_safe_size: 8 +# Master stats enables stats events to be fired from the master at close +# to the defined interval +#master_stats: False +#master_stats_event_iter: 60 + ##### Security settings ##### ########################################## From 704908976272daf060b4eb4a37509774d2095146 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Wed, 29 Nov 2017 19:45:19 -0600 Subject: [PATCH 104/159] Fix calls to verify_env https://github.com/saltstack/salt/pull/44584 deprecated pki_dir but left several references. These lines can be safely removed since sensitive_dirs is already being used in each of these refs. --- salt/cli/daemons.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/salt/cli/daemons.py b/salt/cli/daemons.py index 94e0ff8c13a..e022e4364bb 100644 --- a/salt/cli/daemons.py +++ b/salt/cli/daemons.py @@ -161,7 +161,6 @@ class Master(salt.utils.parsers.MasterOptionParser, DaemonsMixin): # pylint: di v_dirs, self.config['user'], permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], root_dir=self.config['root_dir'], sensitive_dirs=[self.config['pki_dir'], self.config['key_dir']], ) @@ -283,7 +282,6 @@ class Minion(salt.utils.parsers.MinionOptionParser, DaemonsMixin): # pylint: di v_dirs, self.config['user'], permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], root_dir=self.config['root_dir'], sensitive_dirs=[self.config['pki_dir']], ) @@ -472,7 +470,6 @@ class ProxyMinion(salt.utils.parsers.ProxyMinionOptionParser, DaemonsMixin): # v_dirs, self.config['user'], permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], root_dir=self.config['root_dir'], sensitive_dirs=[self.config['pki_dir']], ) @@ -582,7 +579,6 @@ class Syndic(salt.utils.parsers.SyndicOptionParser, DaemonsMixin): # pylint: di ], self.config['user'], permissive=self.config['permissive_pki_access'], - pki_dir=self.config['pki_dir'], root_dir=self.config['root_dir'], sensitive_dirs=[self.config['pki_dir']], ) From 3ebdb5454203c2e56920040e96d9e09a2f4f1ce2 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Thu, 30 Nov 2017 11:04:58 +0000 Subject: [PATCH 105/159] Add Thorium load execution and runner functions --- salt/modules/saltutil.py | 39 +++++++++++++++++++++++++++++++++++++++ salt/runners/saltutil.py | 27 +++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/salt/modules/saltutil.py b/salt/modules/saltutil.py index 12f8be9fa46..0565f83faca 100644 --- a/salt/modules/saltutil.py +++ b/salt/modules/saltutil.py @@ -585,6 +585,44 @@ def sync_engines(saltenv=None, refresh=False, extmod_whitelist=None, extmod_blac return ret +def sync_thorium(saltenv=None, refresh=False, extmod_whitelist=None, extmod_blacklist=None): + ''' + .. versionadded:: Oxygen + + Sync Thorium modules from ``salt://_thorium`` to the minion + + saltenv + The fileserver environment from which to sync. To sync from more than + one environment, pass a comma-separated list. + + If not passed, then all environments configured in the :ref:`top files + ` will be checked for engines to sync. If no top files are + found, then the ``base`` environment will be synced. + + refresh: ``True`` + If ``True``, refresh the available execution modules on the minion. + This refresh will be performed even if no new Thorium modules are synced. + Set to ``False`` to prevent this refresh. + + extmod_whitelist + comma-seperated list of modules to sync + + extmod_blacklist + comma-seperated list of modules to blacklist based on type + + CLI Examples: + + .. code-block:: bash + + salt '*' saltutil.sync_thorium + salt '*' saltutil.sync_thorium saltenv=base,dev + ''' + ret = _sync('thorium', saltenv, extmod_whitelist, extmod_blacklist) + if refresh: + refresh_modules() + return ret + + def sync_output(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist=None): ''' Sync outputters from ``salt://_output`` to the minion @@ -864,6 +902,7 @@ def sync_all(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blacklist ret['log_handlers'] = sync_log_handlers(saltenv, False, extmod_whitelist, extmod_blacklist) ret['proxymodules'] = sync_proxymodules(saltenv, False, extmod_whitelist, extmod_blacklist) ret['engines'] = sync_engines(saltenv, False, extmod_whitelist, extmod_blacklist) + ret['thorium'] = sync_thorium(saltenv, False, extmod_whitelist, extmod_blacklist) if __opts__['file_client'] == 'local': ret['pillar'] = sync_pillar(saltenv, False, extmod_whitelist, extmod_blacklist) if refresh: diff --git a/salt/runners/saltutil.py b/salt/runners/saltutil.py index b691f827e1d..200bfe11a84 100644 --- a/salt/runners/saltutil.py +++ b/salt/runners/saltutil.py @@ -52,6 +52,7 @@ def sync_all(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): ret['runners'] = sync_runners(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['wheel'] = sync_wheel(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['engines'] = sync_engines(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) + ret['thorium'] = sync_thorium(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['queues'] = sync_queues(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['pillar'] = sync_pillar(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['utils'] = sync_utils(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) @@ -303,6 +304,32 @@ def sync_engines(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): extmod_blacklist=extmod_blacklist)[0] +def sync_thorium(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): + ''' + .. versionadded:: Oxygen + + Sync Thorium from ``salt://_thorium`` to the master + + saltenv: ``base`` + The fileserver environment from which to sync. To sync from more than + one environment, pass a comma-separated list. + + extmod_whitelist + comma-seperated list of modules to sync + + extmod_blacklist + comma-seperated list of modules to blacklist based on type + + CLI Example: + + .. code-block:: bash + + salt-run saltutil.sync_thorium + ''' + return salt.utils.extmods.sync(__opts__, 'thorium', saltenv=saltenv, extmod_whitelist=extmod_whitelist, + extmod_blacklist=extmod_blacklist)[0] + + def sync_queues(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): ''' Sync queue modules from ``salt://_queues`` to the master From 0acc8509913d619f7b92acc39c3cee66ee1e926c Mon Sep 17 00:00:00 2001 From: Malte Starostik Date: Thu, 30 Nov 2017 14:08:50 +0100 Subject: [PATCH 106/159] Allow for omission of HWADDR from ifcfg-* file Allow - addr: none in network.managed to omit the HWADDR from the generated ifcfg-* file. This allows to manage interfaces solely by their name, keeping their configuration over MAC changes. For the sake of completeness, also accept - addr: auto to explicitly request the default behaviour with no "addr" specified. --- salt/modules/rh_ip.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/modules/rh_ip.py b/salt/modules/rh_ip.py index b568dd0bb37..668d8f7b0af 100644 --- a/salt/modules/rh_ip.py +++ b/salt/modules/rh_ip.py @@ -643,12 +643,18 @@ def _parse_settings_eth(opts, iface_type, enabled, iface): result[opt] = opts[opt] if iface_type not in ['bond', 'vlan', 'bridge', 'ipip']: + auto_addr = False if 'addr' in opts: if salt.utils.validate.net.mac(opts['addr']): result['addr'] = opts['addr'] - else: - _raise_error_iface(iface, opts['addr'], ['AA:BB:CC:DD:EE:FF']) + elif opts['addr'] == 'auto': + auto_addr = True + elif opts['addr'] != 'none': + _raise_error_iface(iface, opts['addr'], ['AA:BB:CC:DD:EE:FF', 'auto', 'none']) else: + auto_addr = True + + if auto_addr: # If interface type is slave for bond, not setting hwaddr if iface_type != 'slave': ifaces = __salt__['network.interfaces']() From dce017b3f785b554fb0677c7429e58894e1e9ad8 Mon Sep 17 00:00:00 2001 From: Mircea Ulinic Date: Thu, 30 Nov 2017 15:50:11 +0000 Subject: [PATCH 107/159] Make tests happy --- tests/integration/modules/test_saltutil.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/integration/modules/test_saltutil.py b/tests/integration/modules/test_saltutil.py index 1a3647c3bfb..b1db1620b9d 100644 --- a/tests/integration/modules/test_saltutil.py +++ b/tests/integration/modules/test_saltutil.py @@ -93,7 +93,8 @@ class SaltUtilSyncModuleTest(ModuleCase): 'states': [], 'sdb': [], 'proxymodules': [], - 'output': []} + 'output': [], + 'thorium': []} ret = self.run_function('saltutil.sync_all') self.assertEqual(ret, expected_return) @@ -113,7 +114,8 @@ class SaltUtilSyncModuleTest(ModuleCase): 'states': [], 'sdb': [], 'proxymodules': [], - 'output': []} + 'output': [], + 'thorium': []} ret = self.run_function('saltutil.sync_all', extmod_whitelist={'modules': ['salttest']}) self.assertEqual(ret, expected_return) @@ -135,7 +137,8 @@ class SaltUtilSyncModuleTest(ModuleCase): 'states': [], 'sdb': [], 'proxymodules': [], - 'output': []} + 'output': [], + 'thorium': []} ret = self.run_function('saltutil.sync_all', extmod_blacklist={'modules': ['runtests_decorators']}) self.assertEqual(ret, expected_return) @@ -155,7 +158,8 @@ class SaltUtilSyncModuleTest(ModuleCase): 'states': [], 'sdb': [], 'proxymodules': [], - 'output': []} + 'output': [], + 'thorium': []} ret = self.run_function('saltutil.sync_all', extmod_whitelist={'modules': ['runtests_decorators']}, extmod_blacklist={'modules': ['runtests_decorators']}) self.assertEqual(ret, expected_return) From 094d21afb2ed5c1af90841bd58835954873cb107 Mon Sep 17 00:00:00 2001 From: Ari Maniatis Date: Thu, 12 Oct 2017 11:15:19 +1000 Subject: [PATCH 108/159] Better exception After spending an hour trying to figure out why I had a key error in an orchestration state I realised that saltstack requires a runner id with a dot. A clearer error message will save the next person wasting the same time. --- salt/loader.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/loader.py b/salt/loader.py index 81e44ebb27a..4e783e5dffd 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -1588,8 +1588,10 @@ class LazyLoader(salt.utils.lazy.LazyDict): Load a single item if you have it ''' # if the key doesn't have a '.' then it isn't valid for this mod dict - if not isinstance(key, six.string_types) or u'.' not in key: - raise KeyError + if not isinstance(key, six.string_types): + raise KeyError(u'The key must be a string.') + if u'.' not in key: + raise KeyError(u'The key \'%s\' should contain a \'.\'', key) mod_name, _ = key.split(u'.', 1) if mod_name in self.missing_modules: return True From dc3097ceddb5e706bc3a036fe57e6e03e675d471 Mon Sep 17 00:00:00 2001 From: rallytime Date: Fri, 10 Nov 2017 13:41:45 -0500 Subject: [PATCH 109/159] Write a unittest for error message raised with missing "." in function --- tests/unit/loader/test_loader.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/unit/loader/test_loader.py b/tests/unit/loader/test_loader.py index 2403dbba212..c5705144ab0 100644 --- a/tests/unit/loader/test_loader.py +++ b/tests/unit/loader/test_loader.py @@ -278,6 +278,38 @@ class LazyLoaderWhitelistTest(TestCase): self.assertNotIn('grains.get', self.loader) +class LazyLoaderSingleItem(TestCase): + ''' + Test loading a single item via the _load() function + ''' + @classmethod + def setUpClass(cls): + cls.opts = salt.config.minion_config(None) + cls.opts['grains'] = grains(cls.opts) + + def setUp(self): + self.loader = LazyLoader(_module_dirs(copy.deepcopy(self.opts), 'modules', 'module'), + copy.deepcopy(self.opts), + tag='module') + + def tearDown(self): + del self.loader + + def test_single_item_no_dot(self): + ''' + Checks that a KeyError is raised when the function key does not contain a '.' + ''' + with self.assertRaises(KeyError) as err: + inspect.isfunction(self.loader['testing_no_dot']) + + if six.PY2: + self.assertEqual(err.exception[0], + 'The key \'%s\' should contain a \'.\'') + else: + self.assertEqual(str(err.exception), + str(("The key '%s' should contain a '.'", 'testing_no_dot'))) + + module_template = ''' __load__ = ['test', 'test_alias'] __func_alias__ = dict(test_alias='working_alias') From 9e0501c3f02c7614883e7870e0e89463eceb7932 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 30 Nov 2017 09:47:31 -0700 Subject: [PATCH 110/159] initial add of state system and orchestration pauses --- salt/modules/state.py | 50 +++++++++++++++++++++++++++++++++++++++++++ salt/runners/state.py | 18 ++++++++++++++++ salt/state.py | 25 ++++++++++++++++++++++ 3 files changed, 93 insertions(+) diff --git a/salt/modules/state.py b/salt/modules/state.py index af7ce0fe38f..a3f2e5a86fe 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -165,6 +165,56 @@ def _snapper_post(opts, jid, pre_num): log.error('Failed to create snapper pre snapshot for jid: {0}'.format(jid)) +def set_pause(jid, state_id, duration=None): + ''' + Set up a state id pause, this instructs a running state to pause at a given + state id. This needs to pass in the jid of the running state and can + optionally pass in a duration in seconds. + ''' + pause_dir = os.path.join(__opts__[u'cachedir'], 'state_pause') + pause_path = os.path.join(pause_dir, jid) + if not os.path.exists(pause_dir): + try: + os.makedirs(pause_path) + except OSError: + # File created in the gap + pass + data = {} + if os.path.exists(pause_path): + with salt.utils.files.fopen(pause_path, 'rb') as fp_: + data = msgpack.loads(fp_.read()) + if state_id not in data: + data[state_id] = {} + if duration: + data[state_id]['duration'] = int(duration) + with salt.utils.files.fopen(pause_path, 'wb') as fp_: + fp_.write(msgpack.dumps(data)) + + +def rm_pause(jid, state_id): + ''' + Remove a pause from a jid, allowing it to continue + ''' + pause_dir = os.path.join(__opts__[u'cachedir'], 'state_pause') + pause_path = os.path.join(pause_dir, jid) + if not os.path.exists(pause_dir): + try: + os.makedirs(pause_path) + except OSError: + # File created in the gap + pass + data = {} + if os.path.exists(pause_path): + with salt.utils.files.fopen(pause_path, 'rb') as fp_: + data = msgpack.loads(fp_.read()) + else: + return True + if state_id in data: + data.pop(state_id) + with salt.utils.files.fopen(pause_path, 'wb') as fp_: + fp_.write(msgpack.dumps(data)) + + def orchestrate(mods, saltenv='base', test=None, diff --git a/salt/runners/state.py b/salt/runners/state.py index b7399e7b77a..f4f5d9e5c7a 100644 --- a/salt/runners/state.py +++ b/salt/runners/state.py @@ -15,6 +15,24 @@ from salt.exceptions import SaltInvocationError LOGGER = logging.getLogger(__name__) +def set_pause(jid, state_id, duration=None): + ''' + Set up a state id pause, this instructs a running state to pause at a given + state id. This needs to pass in the jid of the running state and can + optionally pass in a duration in seconds. + ''' + minion = salt.minion.MasterMinion(__opts__) + minion['state.set_pause'](jid, state_id, duration) + + +def rm_pause(jid, state_id, duration=None): + ''' + Remove a pause from a jid, allowing it to continue + ''' + minion = salt.minion.MasterMinion(__opts__) + minion['state.rm_pause'](jid, state_id) + + def orchestrate(mods, saltenv='base', test=None, diff --git a/salt/state.py b/salt/state.py index f0e405e77d0..8fca4423ec5 100644 --- a/salt/state.py +++ b/salt/state.py @@ -1918,6 +1918,8 @@ class State(object): if self.mocked: ret = mock_ret(cdata) else: + # Check if this low chunk is paused + self.check_pause(low) # Execute the state function if not low.get(u'__prereq__') and low.get(u'parallel'): # run the state call in parallel, but only if not in a prereq @@ -2127,6 +2129,29 @@ class State(object): return not running[tag][u'result'] return False + def check_pause(self, low): + ''' + Check to see if this low chunk has been paused + ''' + pause_path = os.path.join(self.opts[u'cachedir'], 'state_pause', self.jid) + if os.path.isfile(pause_path): + try: + while True: + with salt.utils.files.fopen(pause_path, 'b') as fp_: + pdat = msgpack.loads(fp_.read()) + id_ = low[u'__id__'] + if id_ in pdat: + if u'duration' in pdat[id_]: + start = pdat[id_]['start'] + now = time.time() + if now - start > pdat[id_][u'duration']: + return + else: + return + except Exception as exc: + log.error('Failed to read in pause data for file located at: %s', pause_path) + return + def reconcile_procs(self, running): ''' Check the running dict for processes and resolve them From 737433b47fb0906cd4a80d241b88cea8a05f6fba Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 30 Nov 2017 11:05:12 -0700 Subject: [PATCH 111/159] It works! --- salt/modules/state.py | 7 +++++-- salt/state.py | 26 +++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/salt/modules/state.py b/salt/modules/state.py index a3f2e5a86fe..f420b518813 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -43,6 +43,7 @@ from salt.runners.state import orchestrate as _orchestrate # Import 3rd-party libs from salt.ext import six +import msgpack __proxyenabled__ = ['*'] @@ -171,11 +172,12 @@ def set_pause(jid, state_id, duration=None): state id. This needs to pass in the jid of the running state and can optionally pass in a duration in seconds. ''' + jid = str(jid) pause_dir = os.path.join(__opts__[u'cachedir'], 'state_pause') pause_path = os.path.join(pause_dir, jid) if not os.path.exists(pause_dir): try: - os.makedirs(pause_path) + os.makedirs(pause_dir) except OSError: # File created in the gap pass @@ -195,11 +197,12 @@ def rm_pause(jid, state_id): ''' Remove a pause from a jid, allowing it to continue ''' + jid = str(jid) pause_dir = os.path.join(__opts__[u'cachedir'], 'state_pause') pause_path = os.path.join(pause_dir, jid) if not os.path.exists(pause_dir): try: - os.makedirs(pause_path) + os.makedirs(pause_dir) except OSError: # File created in the gap pass diff --git a/salt/state.py b/salt/state.py index 8fca4423ec5..71e331487d7 100644 --- a/salt/state.py +++ b/salt/state.py @@ -2134,22 +2134,35 @@ class State(object): Check to see if this low chunk has been paused ''' pause_path = os.path.join(self.opts[u'cachedir'], 'state_pause', self.jid) + start = time.time() if os.path.isfile(pause_path): try: while True: - with salt.utils.files.fopen(pause_path, 'b') as fp_: - pdat = msgpack.loads(fp_.read()) + with salt.utils.files.fopen(pause_path, 'rb') as fp_: + try: + pdat = msgpack.loads(fp_.read()) + except msgpack.UnpackValueError: + # Reading race condition + if tries > 10: + # Break out if there are a ton of read errors + return + tries += 1 + time.sleep(1) + continue id_ = low[u'__id__'] if id_ in pdat: if u'duration' in pdat[id_]: - start = pdat[id_]['start'] now = time.time() + log.error('Duration: %s', pdat[id_][u'duration']) if now - start > pdat[id_][u'duration']: + log.error('now: %s start: %s', now, start) return else: return + time.sleep(1) except Exception as exc: log.error('Failed to read in pause data for file located at: %s', pause_path) + raise return def reconcile_procs(self, running): @@ -2707,6 +2720,13 @@ class State(object): except OSError: log.debug(u'File %s does not exist, no need to cleanup', accum_data_path) _cleanup_accumulator_data() + pause_path = os.path.join(self.opts[u'cachedir'], 'state_pause', self.jid) + if os.path.isfile(pause_path): + try: + os.remove(pause_path) + except OSError: + # File is not present, all is well + pass return ret From 2ab0f06030de0be7ac26ffdfb4e966b8023669a3 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 30 Nov 2017 11:06:55 -0700 Subject: [PATCH 112/159] clean up spurious logs etc --- salt/state.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/salt/state.py b/salt/state.py index 71e331487d7..85cfa26e785 100644 --- a/salt/state.py +++ b/salt/state.py @@ -2153,16 +2153,13 @@ class State(object): if id_ in pdat: if u'duration' in pdat[id_]: now = time.time() - log.error('Duration: %s', pdat[id_][u'duration']) if now - start > pdat[id_][u'duration']: - log.error('now: %s start: %s', now, start) return else: return time.sleep(1) except Exception as exc: log.error('Failed to read in pause data for file located at: %s', pause_path) - raise return def reconcile_procs(self, running): From cb0a0d57cea33ee12316b2a09dee86beb73e6f51 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 30 Nov 2017 11:11:31 -0700 Subject: [PATCH 113/159] Add some basic docs --- salt/modules/state.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/salt/modules/state.py b/salt/modules/state.py index f420b518813..205d3d93e79 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -171,6 +171,23 @@ def set_pause(jid, state_id, duration=None): Set up a state id pause, this instructs a running state to pause at a given state id. This needs to pass in the jid of the running state and can optionally pass in a duration in seconds. + + The given state id is the id got a given state execution, so given a state + that looks like this: + + .. code-block:: yaml + + vim: + pkg.installed: [] + + The state_id to pass to `set_pause` is `vim` + + CLI Examples: + + .. code-block:: bash + + salt '*' state.set_pause 20171130110407769519 vim + salt '*' state.set_pause 20171130110407769519 vim 20 ''' jid = str(jid) pause_dir = os.path.join(__opts__[u'cachedir'], 'state_pause') @@ -196,6 +213,22 @@ def set_pause(jid, state_id, duration=None): def rm_pause(jid, state_id): ''' Remove a pause from a jid, allowing it to continue + + The given state_id is the id got a given state execution, so given a state + that looks like this: + + .. code-block:: yaml + + vim: + pkg.installed: [] + + The state_id to pass to `rm_pause` is `vim` + + CLI Examples: + + .. code-block:: bash + + salt '*' state.rm_pause 20171130110407769519 vim ''' jid = str(jid) pause_dir = os.path.join(__opts__[u'cachedir'], 'state_pause') From bff22e5b0b5d23fdf6acda1b36208725f8b6d29a Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 30 Nov 2017 11:35:33 -0700 Subject: [PATCH 114/159] fix soem bugs and make a better event tag --- salt/config/__init__.py | 2 +- salt/master.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 7b0a8b2ce4a..e095528a28e 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -168,7 +168,7 @@ VALID_OPTS = { # Enable master stats eveents to be fired, these events will contain information about # what commands the master is processing and what the rates are of the executions 'master_stats': bool, - 'master_stats_event_iter': int + 'master_stats_event_iter': int, # The key fingerprint of the higher-level master for the syndic to verify it is talking to the # intended master 'syndic_finger': str, diff --git a/salt/master.py b/salt/master.py index aac98cf0acf..24a3de8e9a4 100644 --- a/salt/master.py +++ b/salt/master.py @@ -890,9 +890,9 @@ class MWorker(salt.utils.process.SignalHandlingMultiprocessingProcess): end = time.time() duration = end - start self.stats[cmd][u'mean'] = (self.stats[cmd][u'mean'] * (self.stats[cmd][u'runs'] - 1) + duration) / self.stats[cmd][u'runs'] - if end - self.stat_clock < self.opts[u'master_stats_event_iter']: + if end - self.stat_clock > self.opts[u'master_stats_event_iter']: # Fire the event with the stats and wipe the tracker - self.aes_funcs.event.fire_event({u'time': end - self.stat_clock, u'worker': self.name, u'stats': self.stats}, tagify(u'stats', u'refresh', u'minion')) + self.aes_funcs.event.fire_event({u'time': end - self.stat_clock, u'worker': self.name, u'stats': self.stats}, tagify(self.name, u'stats')) self.stats = collections.defaultdict(lambda: {'mean': 0, 'runs': 0}) self.stat_clock = end From 52ab561c4cc35ec268a17390e12d7198ac1144cb Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 30 Nov 2017 11:40:40 -0700 Subject: [PATCH 115/159] Add some extra docs --- doc/ref/configuration/master.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index e76bc80a36b..a6f5b555a11 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -868,6 +868,29 @@ what you are doing! Transports are explained in :ref:`Salt Transports ret_port: 4606 zeromq: [] +.. conf_master:: master_stats + +``master_stats`` +---------------- + +Default: False + +Turning on the master stats enables runtime throughput and statistics events +to be fired from the master event bus. These events will report on what +functions have been run on the master and how long these runs have, on +average, taken over a given period of time. + +.. conf_master:: master_stats_event_iter + +``master_stats_event_iter`` +--------------------------- + +Default: 60 + +The time in seconds to fire master_stats events. This will only fire in +conjunction with receiving a request to the master, idle masters will not +fire these events. + .. conf_master:: sock_pool_size ``sock_pool_size`` From 467f96e5641733f85f47ea76fb6fec20126fa4c2 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 30 Nov 2017 14:31:58 -0700 Subject: [PATCH 116/159] add the ability to pause an entire state run This sets the pause at the next state chunk that will run regardless of where the run is right now --- salt/modules/state.py | 8 ++++++-- salt/state.py | 9 +++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/salt/modules/state.py b/salt/modules/state.py index 205d3d93e79..069864818db 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -166,7 +166,7 @@ def _snapper_post(opts, jid, pre_num): log.error('Failed to create snapper pre snapshot for jid: {0}'.format(jid)) -def set_pause(jid, state_id, duration=None): +def set_pause(jid, state_id=None, duration=None): ''' Set up a state id pause, this instructs a running state to pause at a given state id. This needs to pass in the jid of the running state and can @@ -190,6 +190,8 @@ def set_pause(jid, state_id, duration=None): salt '*' state.set_pause 20171130110407769519 vim 20 ''' jid = str(jid) + if state_id is None: + state_id = '__all__' pause_dir = os.path.join(__opts__[u'cachedir'], 'state_pause') pause_path = os.path.join(pause_dir, jid) if not os.path.exists(pause_dir): @@ -210,7 +212,7 @@ def set_pause(jid, state_id, duration=None): fp_.write(msgpack.dumps(data)) -def rm_pause(jid, state_id): +def rm_pause(jid, state_id=None): ''' Remove a pause from a jid, allowing it to continue @@ -231,6 +233,8 @@ def rm_pause(jid, state_id): salt '*' state.rm_pause 20171130110407769519 vim ''' jid = str(jid) + if state_id is None: + state_id = '__all__' pause_dir = os.path.join(__opts__[u'cachedir'], 'state_pause') pause_path = os.path.join(pause_dir, jid) if not os.path.exists(pause_dir): diff --git a/salt/state.py b/salt/state.py index 85cfa26e785..63643321a40 100644 --- a/salt/state.py +++ b/salt/state.py @@ -2150,10 +2150,15 @@ class State(object): time.sleep(1) continue id_ = low[u'__id__'] + key = u'' if id_ in pdat: - if u'duration' in pdat[id_]: + key = id_ + elif u'__all__' in pdat: + key = u'__all__' + if key: + if u'duration' in pdat[key]: now = time.time() - if now - start > pdat[id_][u'duration']: + if now - start > pdat[key][u'duration']: return else: return From ad610fb1505d25690bcb297e2347def92fa3a291 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 30 Nov 2017 14:42:10 -0700 Subject: [PATCH 117/159] fix lint --- salt/modules/state.py | 4 ++-- salt/state.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/modules/state.py b/salt/modules/state.py index 069864818db..227018c467d 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -185,7 +185,7 @@ def set_pause(jid, state_id=None, duration=None): CLI Examples: .. code-block:: bash - + salt '*' state.set_pause 20171130110407769519 vim salt '*' state.set_pause 20171130110407769519 vim 20 ''' @@ -229,7 +229,7 @@ def rm_pause(jid, state_id=None): CLI Examples: .. code-block:: bash - + salt '*' state.rm_pause 20171130110407769519 vim ''' jid = str(jid) diff --git a/salt/state.py b/salt/state.py index 63643321a40..c6449b052a9 100644 --- a/salt/state.py +++ b/salt/state.py @@ -2138,6 +2138,7 @@ class State(object): if os.path.isfile(pause_path): try: while True: + tries = 0 with salt.utils.files.fopen(pause_path, 'rb') as fp_: try: pdat = msgpack.loads(fp_.read()) From 1c50dd28ae42a31d6b558dab06207b9f40a62b6d Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 30 Nov 2017 14:52:16 -0700 Subject: [PATCH 118/159] don't allow pause on salt-ssh --- salt/state.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/state.py b/salt/state.py index c6449b052a9..e94042009f3 100644 --- a/salt/state.py +++ b/salt/state.py @@ -2133,6 +2133,9 @@ class State(object): ''' Check to see if this low chunk has been paused ''' + if not self.jid: + # Can't pause on salt-ssh since we can't track continuous state + return pause_path = os.path.join(self.opts[u'cachedir'], 'state_pause', self.jid) start = time.time() if os.path.isfile(pause_path): From e22596aba7c80a6d646e8196cde758b3bd859e67 Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Thu, 30 Nov 2017 16:13:32 -0700 Subject: [PATCH 119/159] update set_pause to pause and rm_pause to resume --- salt/modules/state.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/salt/modules/state.py b/salt/modules/state.py index 227018c467d..287c353e67f 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -166,11 +166,12 @@ def _snapper_post(opts, jid, pre_num): log.error('Failed to create snapper pre snapshot for jid: {0}'.format(jid)) -def set_pause(jid, state_id=None, duration=None): +def pause(jid, state_id=None, duration=None): ''' Set up a state id pause, this instructs a running state to pause at a given state id. This needs to pass in the jid of the running state and can - optionally pass in a duration in seconds. + optionally pass in a duration in seconds. If a state_id is not passed then + the jid referenced will be paused at the begining of the next state run. The given state id is the id got a given state execution, so given a state that looks like this: @@ -180,14 +181,15 @@ def set_pause(jid, state_id=None, duration=None): vim: pkg.installed: [] - The state_id to pass to `set_pause` is `vim` + The state_id to pass to `pause` is `vim` CLI Examples: .. code-block:: bash - salt '*' state.set_pause 20171130110407769519 vim - salt '*' state.set_pause 20171130110407769519 vim 20 + salt '*' state.pause 20171130110407769519 + salt '*' state.pause 20171130110407769519 vim + salt '*' state.pause 20171130110407769519 vim 20 ''' jid = str(jid) if state_id is None: @@ -212,9 +214,10 @@ def set_pause(jid, state_id=None, duration=None): fp_.write(msgpack.dumps(data)) -def rm_pause(jid, state_id=None): +def resume(jid, state_id=None): ''' - Remove a pause from a jid, allowing it to continue + Remove a pause from a jid, allowing it to continue. If the state_id is + not specified then the a general pause will be resumed. The given state_id is the id got a given state execution, so given a state that looks like this: @@ -230,7 +233,8 @@ def rm_pause(jid, state_id=None): .. code-block:: bash - salt '*' state.rm_pause 20171130110407769519 vim + salt '*' state.resume 20171130110407769519 + salt '*' state.resume 20171130110407769519 vim ''' jid = str(jid) if state_id is None: From 52c2366c232444c90675d22e550093aceac87c6a Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Mon, 6 Nov 2017 22:29:56 -0700 Subject: [PATCH 120/159] addding not_during_range to schedule --- salt/modules/schedule.py | 3 ++- salt/utils/schedule.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/salt/modules/schedule.py b/salt/modules/schedule.py index 487713c5ca1..29e91398682 100644 --- a/salt/modules/schedule.py +++ b/salt/modules/schedule.py @@ -58,6 +58,7 @@ SCHEDULE_CONF = [ 'return_config', 'return_kwargs', 'run_on_start' + 'not_during_range', ] @@ -353,7 +354,7 @@ def build_schedule_item(name, **kwargs): for item in ['range', 'when', 'once', 'once_fmt', 'cron', 'returner', 'after', 'return_config', 'return_kwargs', - 'until', 'run_on_start']: + 'until', 'run_on_start', 'not_during_range']: if item in kwargs: schedule[name][item] = kwargs[item] diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 8e9ecec0a27..d7e65d7b7dd 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -1322,6 +1322,37 @@ class Schedule(object): Ignoring job {0}.'.format(job)) continue + if 'not_during_range' in data: + if not _RANGE_SUPPORTED: + log.error('Missing python-dateutil. Ignoring job {0}'.format(job)) + continue + else: + if isinstance(data['not_during_range'], dict): + try: + start = int(time.mktime(dateutil_parser.parse(data['not_during_range']['start']).timetuple())) + except ValueError: + log.error('Invalid date string for start in not_during_range. Ignoring job {0}.'.format(job)) + continue + try: + end = int(time.mktime(dateutil_parser.parse(data['not_during_range']['end']).timetuple())) + except ValueError: + log.error('Invalid date string for end in not_during_range. Ignoring job {0}.'.format(job)) + log.error(data) + continue + if end > start: + if start <= now <= end: + run = False + else: + run = True + else: + log.error('schedule.handle_func: Invalid range, end must be larger than start. \ + Ignoring job {0}.'.format(job)) + continue + else: + log.error('schedule.handle_func: Invalid, range must be specified as a dictionary. \ + Ignoring job {0}.'.format(job)) + continue + if not run: continue From be192ff7fc9add5b8b74b2e37bbf73df11d23ba8 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 21 Nov 2017 16:42:51 -0800 Subject: [PATCH 121/159] Renaming not_during_range to skip_during_range, adding now as a parameter for eval to override the current time. Adding skip_explicit to include an explicity list of times in unix time stamp format when jobs should be skipped. Adding ability to include a function that should be run if jobs are to be skipped. --- salt/modules/schedule.py | 4 +-- salt/utils/schedule.py | 56 +++++++++++++++++++++++++++++++++------- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/salt/modules/schedule.py b/salt/modules/schedule.py index 29e91398682..06bc92308df 100644 --- a/salt/modules/schedule.py +++ b/salt/modules/schedule.py @@ -58,7 +58,7 @@ SCHEDULE_CONF = [ 'return_config', 'return_kwargs', 'run_on_start' - 'not_during_range', + 'skip_during_range', ] @@ -354,7 +354,7 @@ def build_schedule_item(name, **kwargs): for item in ['range', 'when', 'once', 'once_fmt', 'cron', 'returner', 'after', 'return_config', 'return_kwargs', - 'until', 'run_on_start', 'not_during_range']: + 'until', 'run_on_start', 'skip_during_range']: if item in kwargs: schedule[name][item] = kwargs[item] diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index d7e65d7b7dd..16aa94c09c5 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -409,6 +409,7 @@ class Schedule(object): self.proxy = proxy self.functions = functions self.standalone = standalone + self.skip_function = None if isinstance(intervals, dict): self.intervals = intervals else: @@ -948,7 +949,7 @@ class Schedule(object): # Let's make sure we exit the process! sys.exit(salt.defaults.exitcodes.EX_GENERIC) - def eval(self): + def eval(self, now=None): ''' Evaluate and execute the schedule ''' @@ -974,9 +975,13 @@ class Schedule(object): raise ValueError('Schedule must be of type dict.') if 'enabled' in schedule and not schedule['enabled']: return + if 'skip_function' in schedule: + self.skip_function = schedule['skip_function'] for job, data in six.iteritems(schedule): if job == 'enabled' or not data: continue + if job == 'skip_function' or not data: + continue if not isinstance(data, dict): log.error('Scheduled job "{0}" should have a dict value, not {1}'.format(job, type(data))) continue @@ -1011,7 +1016,8 @@ class Schedule(object): '_run_on_start' not in data: data['_run_on_start'] = True - now = int(time.time()) + if not now: + now = int(time.time()) if 'until' in data: if not _WHEN_SUPPORTED: @@ -1312,7 +1318,11 @@ class Schedule(object): if start <= now <= end: run = True else: - run = False + if self.skip_function: + run = True + func = self.skip_function + else: + run = False else: log.error('schedule.handle_func: Invalid range, end must be larger than start. \ Ignoring job {0}.'.format(job)) @@ -1322,26 +1332,30 @@ class Schedule(object): Ignoring job {0}.'.format(job)) continue - if 'not_during_range' in data: + if 'skip_during_range' in data: if not _RANGE_SUPPORTED: log.error('Missing python-dateutil. Ignoring job {0}'.format(job)) continue else: - if isinstance(data['not_during_range'], dict): + if isinstance(data['skip_during_range'], dict): try: - start = int(time.mktime(dateutil_parser.parse(data['not_during_range']['start']).timetuple())) + start = int(time.mktime(dateutil_parser.parse(data['skip_during_range']['start']).timetuple())) except ValueError: - log.error('Invalid date string for start in not_during_range. Ignoring job {0}.'.format(job)) + log.error('Invalid date string for start in skip_during_range. Ignoring job {0}.'.format(job)) continue try: - end = int(time.mktime(dateutil_parser.parse(data['not_during_range']['end']).timetuple())) + end = int(time.mktime(dateutil_parser.parse(data['skip_during_range']['end']).timetuple())) except ValueError: - log.error('Invalid date string for end in not_during_range. Ignoring job {0}.'.format(job)) + log.error('Invalid date string for end in skip_during_range. Ignoring job {0}.'.format(job)) log.error(data) continue if end > start: if start <= now <= end: - run = False + if self.skip_function: + run = True + func = self.skip_function + else: + run = False else: run = True else: @@ -1353,6 +1367,28 @@ class Schedule(object): Ignoring job {0}.'.format(job)) continue + if 'skip_explicit' in data: + _skip_explicit = data['skip_explicit'] + + if isinstance(_skip_explicit, six.string_types): + _skip_explicit = [_skip_explicit] + + # Copy the list so we can loop through it + for i in copy.deepcopy(_skip_explicit): + if len(_skip_explicit) > 1: + if i < now - self.opts['loop_interval']: + _skip_explicit.remove(i) + + if _skip_explicit: + if _skip_explicit[0] <= now <= (_skip_explicit[0] + self.opts['loop_interval']): + if self.skip_function: + run = True + func = self.skip_function + else: + run = False + else: + run = True + if not run: continue From e4f5b3026a684524a99b0f7b9117e02bcbfd93f2 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Tue, 28 Nov 2017 12:38:32 -0800 Subject: [PATCH 122/159] Adding ability to postpone jobs usiing a simple, initial postpone function. The postpone function adds the current job time to a skip_explicit list and adds the new run time to a run_explicit list. --- salt/minion.py | 4 ++ salt/modules/schedule.py | 100 ++++++++++++++++++++++++++++++++++++++- salt/utils/schedule.py | 62 ++++++++++++++++++++++-- 3 files changed, 162 insertions(+), 4 deletions(-) diff --git a/salt/minion.py b/salt/minion.py index bd4cfcc01b2..40eec5abcf5 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -2067,12 +2067,16 @@ class Minion(MinionBase): self.schedule.run_job(name) elif func == u'disable_job': self.schedule.disable_job(name, persist) + elif func == u'postpone_job': + self.schedule.postpone_job(name, data) elif func == u'reload': self.schedule.reload(schedule) elif func == u'list': self.schedule.list(where) elif func == u'save_schedule': self.schedule.save_schedule() + elif func == u'get_next_fire_time': + self.schedule.get_next_fire_time(name) def manage_beacons(self, tag, data): ''' diff --git a/salt/modules/schedule.py b/salt/modules/schedule.py index 06bc92308df..2e121f0bfe1 100644 --- a/salt/modules/schedule.py +++ b/salt/modules/schedule.py @@ -10,6 +10,7 @@ Module for managing the Salt schedule on a minion from __future__ import absolute_import import copy as pycopy import difflib +import logging import os import yaml @@ -23,7 +24,6 @@ from salt.ext import six __proxyenabled__ = ['*'] -import logging log = logging.getLogger(__name__) __func_alias__ = { @@ -952,3 +952,101 @@ def copy(name, target, **kwargs): ret['minions'] = minions return ret return ret + + +def postpone_job(name, time, new_time, **kwargs): + ''' + Postpone a job in the minion's schedule + + Current time and new time should be specified as Unix timestamps + + CLI Example: + + .. code-block:: bash + + salt '*' schedule.postpone_job current_time new_time + ''' + + ret = {'comment': [], + 'result': True} + + if not name: + ret['comment'] = 'Job name is required.' + ret['result'] = False + + if 'test' in __opts__ and __opts__['test']: + ret['comment'] = 'Job: {0} would be postponed in schedule.'.format(name) + else: + + if name in list_(show_all=True, where='opts', return_yaml=False): + event_data = {'name': name, + 'time': time, + 'new_time': new_time, + 'func': 'postpone_job'} + elif name in list_(show_all=True, where='pillar', return_yaml=False): + event_data = {'name': name, + 'time': time, + 'new_time': new_time, + 'where': 'pillar', + 'func': 'postpone_job'} + else: + ret['comment'] = 'Job {0} does not exist.'.format(name) + ret['result'] = False + return ret + + try: + eventer = salt.utils.event.get_event('minion', opts=__opts__) + res = __salt__['event.fire'](event_data, 'manage_schedule') + if res: + event_ret = eventer.get_event(tag='/salt/minion/minion_schedule_postpone_job_complete', wait=30) + if event_ret and event_ret['complete']: + schedule = event_ret['schedule'] + # check item exists in schedule and is enabled + if name in schedule and schedule[name]['enabled']: + ret['result'] = True + ret['comment'] = 'Postponed Job {0} in schedule.'.format(name) + else: + ret['result'] = False + ret['comment'] = 'Failed to postpone job {0} in schedule.'.format(name) + return ret + except KeyError: + # Effectively a no-op, since we can't really return without an event system + ret['comment'] = 'Event module not available. Schedule postpone job failed.' + return ret + + +def show_next_fire_time(name, **kwargs): + ''' + Show the next fire time for scheduled job + + CLI Example: + + .. code-block:: bash + + salt '*' schedule.show_next_fire_time job_name + + ''' + + ret = {'comment': [], + 'result': True} + + if not name: + ret['comment'] = 'Job name is required.' + ret['result'] = False + + try: + event_data = {'name': name, 'func': 'get_next_fire_time'} + eventer = salt.utils.event.get_event('minion', opts=__opts__) + res = __salt__['event.fire'](event_data, + 'manage_schedule') + if res: + event_ret = eventer.get_event(tag='/salt/minion/minion_schedule_next_fire_time_complete', wait=30) + except KeyError: + # Effectively a no-op, since we can't really return without an event system + ret = {} + ret['comment'] = 'Event module not available. Schedule show next fire time failed.' + ret['result'] = True + log.debug(ret['comment']) + return ret + + return event_ret diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 16aa94c09c5..7bca1f1a4c1 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -746,6 +746,46 @@ class Schedule(object): evt.fire_event({'complete': True}, tag='/salt/minion/minion_schedule_saved') + def postpone_job(self, name, data): + ''' + Disable a job in the scheduler. Ignores jobs from pillar + ''' + time = data['time'] + new_time = data['new_time'] + + # ensure job exists, then disable it + if name in self.opts['schedule']: + if 'skip_explicit' not in self.opts['schedule'][name]: + self.opts['schedule'][name]['skip_explicit'] = [] + self.opts['schedule'][name]['skip_explicit'].append(time) + + if 'run_explicit' not in self.opts['schedule'][name]: + self.opts['schedule'][name]['run_explicit'] = [] + self.opts['schedule'][name]['run_explicit'].append(new_time) + + elif name in self._get_schedule(include_opts=False): + log.warning('Cannot modify job {0}, ' + 'it`s in the pillar!'.format(name)) + + # Fire the complete event back along with updated list of schedule + evt = salt.utils.event.get_event('minion', opts=self.opts, listen=False) + evt.fire_event({'complete': True, 'schedule': self._get_schedule()}, + tag='/salt/minion/minion_schedule_postpone_job_complete') + + def get_next_fire_time(self, name): + ''' + Disable a job in the scheduler. Ignores jobs from pillar + ''' + + schedule = self._get_schedule() + if schedule: + _next_fire_time = schedule[name]['_next_fire_time'] + + # Fire the complete event back along with updated list of schedule + evt = salt.utils.event.get_event('minion', opts=self.opts, listen=False) + evt.fire_event({'complete': True, 'next_fire_time': _next_fire_time}, + tag='/salt/minion/minion_schedule_next_fire_time_complete') + def handle_func(self, multiprocessing_enabled, func, data): ''' Execute this method in a multiprocess or thread @@ -1071,6 +1111,23 @@ class Schedule(object): '", "'.join(scheduling_elements))) continue + if 'run_explicit' in data: + _run_explicit = data['run_explicit'] + + if isinstance(_run_explicit, six.string_types): + _run_explicit = [_run_explicit] + + # Copy the list so we can loop through it + for i in copy.deepcopy(_run_explicit): + if len(_run_explicit) > 1: + if i < now - self.opts['loop_interval']: + _run_explicit.remove(i) + + if _run_explicit: + if _run_explicit[0] <= now <= (_run_explicit[0] + self.opts['loop_interval']): + run = True + data['_next_fire_time'] = _run_explicit[0] + if True in [True for item in time_elements if item in data]: if '_seconds' not in data: interval = int(data.get('seconds', 0)) @@ -1375,9 +1432,8 @@ class Schedule(object): # Copy the list so we can loop through it for i in copy.deepcopy(_skip_explicit): - if len(_skip_explicit) > 1: - if i < now - self.opts['loop_interval']: - _skip_explicit.remove(i) + if i < now - self.opts['loop_interval']: + _skip_explicit.remove(i) if _skip_explicit: if _skip_explicit[0] <= now <= (_skip_explicit[0] + self.opts['loop_interval']): From a28e39ef31ce39c703cb656cc2e1ee3b3af049ff Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 29 Nov 2017 12:09:58 -0800 Subject: [PATCH 123/159] Merging changes from #44630 for updates to scheduler to deal with loop_interval. --- salt/utils/schedule.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 7bca1f1a4c1..4681cd713eb 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -994,6 +994,8 @@ class Schedule(object): Evaluate and execute the schedule ''' + log.trace('==== evaluating schedule =====') + def _splay(splaytime): ''' Calculate splaytime @@ -1216,10 +1218,11 @@ class Schedule(object): # Copy the list so we can loop through it for i in copy.deepcopy(_when): - if i < now and len(_when) > 1: - # Remove all missed schedules except the latest one. - # We need it to detect if it was triggered previously. - _when.remove(i) + if len(_when) > 1: + if i < now - self.opts['loop_interval']: + # Remove all missed schedules except the latest one. + # We need it to detect if it was triggered previously. + _when.remove(i) if _when: # Grab the first element, which is the next run time or @@ -1321,19 +1324,21 @@ class Schedule(object): seconds = data['_next_fire_time'] - now if data['_splay']: seconds = data['_splay'] - now - if seconds <= 0: - if '_seconds' in data: + if '_seconds' in data: + if seconds <= 0: run = True - elif 'when' in data and data['_run']: + elif 'when' in data and data['_run']: + if data['_next_fire_time'] <= now <= (data['_next_fire_time'] + self.opts['loop_interval']): data['_run'] = False run = True - elif 'cron' in data: - # Reset next scheduled time because it is in the past now, - # and we should trigger the job run, then wait for the next one. + elif 'cron' in data: + # Reset next scheduled time because it is in the past now, + # and we should trigger the job run, then wait for the next one. + if seconds <= 0: data['_next_fire_time'] = None run = True - elif seconds == 0: - run = True + elif seconds == 0: + run = True if '_run_on_start' in data and data['_run_on_start']: run = True @@ -1497,6 +1502,7 @@ class Schedule(object): finally: if '_seconds' in data: data['_next_fire_time'] = now + data['_seconds'] + data['_last_run'] = now data['_splay'] = None if salt.utils.platform.is_windows(): # Restore our function references. From b681c229a018d1af481a6c62490b8aee75ecdace Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Wed, 29 Nov 2017 13:58:38 -0800 Subject: [PATCH 124/159] Adding a simple job to skip jobs at specified times. Adding versionadded tags to new functions. Updating parameters for postpone function to ensure it's more clear. Adding documentation for which was implemented awhile ago but never documented. --- salt/modules/schedule.py | 85 ++++++++++++++++++++++++++++++++++++++-- salt/utils/schedule.py | 51 +++++++++++++++++++++++- 2 files changed, 130 insertions(+), 6 deletions(-) diff --git a/salt/modules/schedule.py b/salt/modules/schedule.py index 2e121f0bfe1..efb745e3cb4 100644 --- a/salt/modules/schedule.py +++ b/salt/modules/schedule.py @@ -954,17 +954,19 @@ def copy(name, target, **kwargs): return ret -def postpone_job(name, time, new_time, **kwargs): +def postpone_job(name, current_time, new_time, **kwargs): ''' Postpone a job in the minion's schedule Current time and new time should be specified as Unix timestamps + .. versionadded:: Oxygen + CLI Example: .. code-block:: bash - salt '*' schedule.postpone_job current_time new_time + salt '*' schedule.postpone_job job current_time new_time ''' ret = {'comment': [], @@ -974,18 +976,26 @@ def postpone_job(name, time, new_time, **kwargs): ret['comment'] = 'Job name is required.' ret['result'] = False + if not current_time: + ret['comment'] = 'Job current time is required.' + ret['result'] = False + + if not new_time: + ret['comment'] = 'Job new_time is required.' + ret['result'] = False + if 'test' in __opts__ and __opts__['test']: ret['comment'] = 'Job: {0} would be postponed in schedule.'.format(name) else: if name in list_(show_all=True, where='opts', return_yaml=False): event_data = {'name': name, - 'time': time, + 'time': current_time, 'new_time': new_time, 'func': 'postpone_job'} elif name in list_(show_all=True, where='pillar', return_yaml=False): event_data = {'name': name, - 'time': time, + 'time': current_time, 'new_time': new_time, 'where': 'pillar', 'func': 'postpone_job'} @@ -1015,10 +1025,77 @@ def postpone_job(name, time, new_time, **kwargs): return ret +def skip_job(name, time, **kwargs): + ''' + Skip a job in the minion's schedule at specified time. + + Time to skip should be specified as Unix timestamps + + .. versionadded:: Oxygen + + CLI Example: + + .. code-block:: bash + + salt '*' schedule.skip_job job time + ''' + + ret = {'comment': [], + 'result': True} + + if not name: + ret['comment'] = 'Job name is required.' + ret['result'] = False + + if not time: + ret['comment'] = 'Job time is required.' + ret['result'] = False + + if 'test' in __opts__ and __opts__['test']: + ret['comment'] = 'Job: {0} would be skipped in schedule.'.format(name) + else: + + if name in list_(show_all=True, where='opts', return_yaml=False): + event_data = {'name': name, + 'time': time, + 'func': 'skip_job'} + elif name in list_(show_all=True, where='pillar', return_yaml=False): + event_data = {'name': name, + 'time': time, + 'where': 'pillar', + 'func': 'skip_job'} + else: + ret['comment'] = 'Job {0} does not exist.'.format(name) + ret['result'] = False + return ret + + try: + eventer = salt.utils.event.get_event('minion', opts=__opts__) + res = __salt__['event.fire'](event_data, 'manage_schedule') + if res: + event_ret = eventer.get_event(tag='/salt/minion/minion_schedule_skip_job_complete', wait=30) + if event_ret and event_ret['complete']: + schedule = event_ret['schedule'] + # check item exists in schedule and is enabled + if name in schedule and schedule[name]['enabled']: + ret['result'] = True + ret['comment'] = 'Added Skip Job {0} in schedule.'.format(name) + else: + ret['result'] = False + ret['comment'] = 'Failed to skip job {0} in schedule.'.format(name) + return ret + except KeyError: + # Effectively a no-op, since we can't really return without an event system + ret['comment'] = 'Event module not available. Schedule skip job failed.' + return ret + + def show_next_fire_time(name, **kwargs): ''' Show the next fire time for scheduled job + .. versionadded:: Oxygen + CLI Example: .. code-block:: bash diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 4681cd713eb..0a05702210a 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -89,6 +89,28 @@ localtime. This will schedule the command: ``state.sls httpd test=True`` at 5:00 PM on Monday, Wednesday and Friday, and 3:00 PM on Tuesday and Thursday. +.. code-block:: yaml + + schedule: + job1: + function: state.sls + args: + - httpd + kwargs: + test: True + when: + - 'tea time' + +.. code-block:: yaml + + whens: + tea time: 1:40pm + deployment time: Friday 5:00pm + +The Salt scheduler also allows custom phrases to be used for the `when` +parameter. These `whens` can be stored as either pillar values or +grain values. + .. code-block:: yaml schedule: @@ -333,7 +355,6 @@ import logging import errno import random import yaml -import copy # Import Salt libs import salt.config @@ -748,7 +769,8 @@ class Schedule(object): def postpone_job(self, name, data): ''' - Disable a job in the scheduler. Ignores jobs from pillar + Postpone a job in the scheduler. + Ignores jobs from pillar ''' time = data['time'] new_time = data['new_time'] @@ -772,6 +794,28 @@ class Schedule(object): evt.fire_event({'complete': True, 'schedule': self._get_schedule()}, tag='/salt/minion/minion_schedule_postpone_job_complete') + def skip_job(self, name, data): + ''' + Skip a job at a specific time in the scheduler. + Ignores jobs from pillar + ''' + time = data['time'] + + # ensure job exists, then disable it + if name in self.opts['schedule']: + if 'skip_explicit' not in self.opts['schedule'][name]: + self.opts['schedule'][name]['skip_explicit'] = [] + self.opts['schedule'][name]['skip_explicit'].append(time) + + elif name in self._get_schedule(include_opts=False): + log.warning('Cannot modify job {0}, ' + 'it`s in the pillar!'.format(name)) + + # Fire the complete event back along with updated list of schedule + evt = salt.utils.event.get_event('minion', opts=self.opts, listen=False) + evt.fire_event({'complete': True, 'schedule': self._get_schedule()}, + tag='/salt/minion/minion_schedule_skip_job_complete') + def get_next_fire_time(self, name): ''' Disable a job in the scheduler. Ignores jobs from pillar @@ -1248,16 +1292,19 @@ class Schedule(object): else: if ('pillar' in self.opts and 'whens' in self.opts['pillar'] and data['when'] in self.opts['pillar']['whens']): + log.debug('=== whens found in pillar ===') if not isinstance(self.opts['pillar']['whens'], dict): log.error('Pillar item "whens" must be dict.' 'Ignoring') continue _when = self.opts['pillar']['whens'][data['when']] + log.debug('=== _when {} ==='.format(_when)) try: when__ = dateutil_parser.parse(_when) except ValueError: log.error('Invalid date string. Ignoring') continue + log.debug('=== __when {} ==='.format(__when)) elif ('whens' in self.opts['grains'] and data['when'] in self.opts['grains']['whens']): if not isinstance(self.opts['grains']['whens'], dict): From dc2a0313e0e6f993bbfc231ab2fbf7c8638c7c10 Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 30 Nov 2017 09:44:17 -0800 Subject: [PATCH 125/159] Adding requested changes for validation and documentation --- salt/modules/schedule.py | 13 +++++++++++++ salt/utils/schedule.py | 6 +++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/salt/modules/schedule.py b/salt/modules/schedule.py index efb745e3cb4..d4a83651b44 100644 --- a/salt/modules/schedule.py +++ b/salt/modules/schedule.py @@ -975,14 +975,27 @@ def postpone_job(name, current_time, new_time, **kwargs): if not name: ret['comment'] = 'Job name is required.' ret['result'] = False + return ret if not current_time: ret['comment'] = 'Job current time is required.' ret['result'] = False + return ret + else: + if not isinstance(current_time, six.integer_types): + ret['comment'] = 'Job current time must be an integer.' + ret['result'] = False + return ret if not new_time: ret['comment'] = 'Job new_time is required.' ret['result'] = False + return ret + else: + if not isinstance(new_time, six.integer_types): + ret['comment'] = 'Job new time must be an integer.' + ret['result'] = False + return ret if 'test' in __opts__ and __opts__['test']: ret['comment'] = 'Job: {0} would be postponed in schedule.'.format(name) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 0a05702210a..7b9734efcb8 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -1036,6 +1036,9 @@ class Schedule(object): def eval(self, now=None): ''' Evaluate and execute the schedule + + :param int now: Override current time with a Unix timestamp`` + ''' log.trace('==== evaluating schedule =====') @@ -1292,19 +1295,16 @@ class Schedule(object): else: if ('pillar' in self.opts and 'whens' in self.opts['pillar'] and data['when'] in self.opts['pillar']['whens']): - log.debug('=== whens found in pillar ===') if not isinstance(self.opts['pillar']['whens'], dict): log.error('Pillar item "whens" must be dict.' 'Ignoring') continue _when = self.opts['pillar']['whens'][data['when']] - log.debug('=== _when {} ==='.format(_when)) try: when__ = dateutil_parser.parse(_when) except ValueError: log.error('Invalid date string. Ignoring') continue - log.debug('=== __when {} ==='.format(__when)) elif ('whens' in self.opts['grains'] and data['when'] in self.opts['grains']['whens']): if not isinstance(self.opts['grains']['whens'], dict): From ac8f28d921c5a022b28b447c3f8f991e1256d95d Mon Sep 17 00:00:00 2001 From: "Gareth J. Greenaway" Date: Thu, 30 Nov 2017 22:01:37 -0800 Subject: [PATCH 126/159] fixing issue with jobs running twice. --- salt/utils/schedule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index 7b9734efcb8..3261a367287 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -1173,7 +1173,7 @@ class Schedule(object): _run_explicit.remove(i) if _run_explicit: - if _run_explicit[0] <= now <= (_run_explicit[0] + self.opts['loop_interval']): + if _run_explicit[0] <= now < (_run_explicit[0] + self.opts['loop_interval']): run = True data['_next_fire_time'] = _run_explicit[0] From 6ea51d8bcd198f91453a47594072a67460dd0d72 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 27 Nov 2017 10:58:26 +0100 Subject: [PATCH 127/159] Fix broken Python indent. --- salt/modules/ansiblegate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/ansiblegate.py b/salt/modules/ansiblegate.py index d2e952c192b..21e0ac5aae9 100644 --- a/salt/modules/ansiblegate.py +++ b/salt/modules/ansiblegate.py @@ -221,7 +221,7 @@ def __virtual__(): global _caller _resolver = AnsibleModuleResolver(__opts__).resolve().install() _caller = AnsibleModuleCaller(_resolver) - _set_callables(list()) + _set_callables(list()) return ret, msg From fa21af69ddcfb78bf39ac54ff7575f8c4fc66d5e Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 27 Nov 2017 10:58:48 +0100 Subject: [PATCH 128/159] Add unit test for __virtual__ in case Ansible is not installed on the Minion --- tests/unit/modules/test_ansiblegate.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/modules/test_ansiblegate.py b/tests/unit/modules/test_ansiblegate.py index 61de9786de7..4b46e5c62fd 100644 --- a/tests/unit/modules/test_ansiblegate.py +++ b/tests/unit/modules/test_ansiblegate.py @@ -126,3 +126,11 @@ description: patch('salt.modules.ansiblegate.importlib.import_module', lambda x: x): with pytest.raises(LoaderError) as loader_error: self.resolver.load_module('something.strange') + + def test_virtual_function_no_ansible_installed(self): + ''' + Test Ansible module __virtual__ when ansible is not installed on the minion. + :return: + ''' + with patch('salt.modules.ansiblegate.ansible', None): + assert ansible.__virtual__() == (False, 'Ansible is not installed on this system') From e1eec2c69d2012704d78f95b8d8e181a16f42f4f Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 27 Nov 2017 11:11:04 +0100 Subject: [PATCH 129/159] Add unit test for __virtual__ in case Ansible is installed on the Minion --- tests/unit/modules/test_ansiblegate.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/unit/modules/test_ansiblegate.py b/tests/unit/modules/test_ansiblegate.py index 4b46e5c62fd..00df52f6746 100644 --- a/tests/unit/modules/test_ansiblegate.py +++ b/tests/unit/modules/test_ansiblegate.py @@ -134,3 +134,18 @@ description: ''' with patch('salt.modules.ansiblegate.ansible', None): assert ansible.__virtual__() == (False, 'Ansible is not installed on this system') + + @patch('salt.modules.ansiblegate.ansible', MagicMock()) + @patch('salt.modules.ansiblegate.list', MagicMock()) + @patch('salt.modules.ansiblegate._set_callables', MagicMock()) + @patch('salt.modules.ansiblegate.AnsibleModuleCaller', MagicMock()) + def test_virtual_function_ansible_is_installed(self): + ''' + Test Ansible module __virtual__ when ansible is installed on the minion. + :return: + ''' + resolver = MagicMock() + resolver.resolve = MagicMock() + resolver.resolve.install = MagicMock() + with patch('salt.modules.ansiblegate.AnsibleModuleResolver', resolver): + assert ansible.__virtual__() == (True, None) From 150dcc8b90ee792099539634acd9a1e36336c60c Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 27 Nov 2017 11:15:05 +0100 Subject: [PATCH 130/159] Move logging about load failure to the debug level --- salt/modules/ansiblegate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/ansiblegate.py b/salt/modules/ansiblegate.py index 21e0ac5aae9..56dc80c8910 100644 --- a/salt/modules/ansiblegate.py +++ b/salt/modules/ansiblegate.py @@ -215,7 +215,7 @@ def __virtual__(): ret = ansible is not None msg = not ret and "Ansible is not installed on this system" or None if msg: - log.warning(msg) + log.debug(msg) else: global _resolver global _caller From cfe67fdecd5fae78fdac1472d52664a3b7c9a914 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Mon, 27 Nov 2017 23:49:21 +0100 Subject: [PATCH 131/159] Move __virtual__ logging into trace level --- salt/modules/ansiblegate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/modules/ansiblegate.py b/salt/modules/ansiblegate.py index 56dc80c8910..d8b55b22abf 100644 --- a/salt/modules/ansiblegate.py +++ b/salt/modules/ansiblegate.py @@ -215,7 +215,7 @@ def __virtual__(): ret = ansible is not None msg = not ret and "Ansible is not installed on this system" or None if msg: - log.debug(msg) + log.trace(msg) else: global _resolver global _caller From dc9150691313fdf825e9df7be0cc5fa98545e819 Mon Sep 17 00:00:00 2001 From: Bo Maryniuk Date: Thu, 30 Nov 2017 15:31:41 +0100 Subject: [PATCH 132/159] Remove logging from the trace level in __virtual__ --- salt/modules/ansiblegate.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/salt/modules/ansiblegate.py b/salt/modules/ansiblegate.py index d8b55b22abf..8e59d61402a 100644 --- a/salt/modules/ansiblegate.py +++ b/salt/modules/ansiblegate.py @@ -214,9 +214,7 @@ def __virtual__(): ''' ret = ansible is not None msg = not ret and "Ansible is not installed on this system" or None - if msg: - log.trace(msg) - else: + if ret: global _resolver global _caller _resolver = AnsibleModuleResolver(__opts__).resolve().install() From 4abc8235a2287143f6b8dc88f2e7f6fc806f96fc Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 1 Dec 2017 08:18:28 -0700 Subject: [PATCH 133/159] Add extra check for non-jid runs, as these can't be paused --- salt/state.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/salt/state.py b/salt/state.py index e94042009f3..e7886231e4e 100644 --- a/salt/state.py +++ b/salt/state.py @@ -2726,13 +2726,14 @@ class State(object): except OSError: log.debug(u'File %s does not exist, no need to cleanup', accum_data_path) _cleanup_accumulator_data() - pause_path = os.path.join(self.opts[u'cachedir'], 'state_pause', self.jid) - if os.path.isfile(pause_path): - try: - os.remove(pause_path) - except OSError: - # File is not present, all is well - pass + if self.jid is not None: + pause_path = os.path.join(self.opts[u'cachedir'], u'state_pause', self.jid) + if os.path.isfile(pause_path): + try: + os.remove(pause_path) + except OSError: + # File is not present, all is well + pass return ret From d2ccd6acd621b3f830f08aea826758c05892cebd Mon Sep 17 00:00:00 2001 From: Thomas S Hatch Date: Fri, 1 Dec 2017 12:02:15 -0700 Subject: [PATCH 134/159] try to ensure that the ret is never None --- salt/returners/local_cache.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/salt/returners/local_cache.py b/salt/returners/local_cache.py index c0b135c3efa..900eaca0f73 100644 --- a/salt/returners/local_cache.py +++ b/salt/returners/local_cache.py @@ -294,9 +294,11 @@ def get_load(jid): if not os.path.exists(jid_dir) or not os.path.exists(load_fn): return {} serial = salt.payload.Serial(__opts__) + ret = {} with salt.utils.files.fopen(os.path.join(jid_dir, LOAD_P), 'rb') as rfh: ret = serial.load(rfh) - + if ret is None: + ret = {} minions_cache = [os.path.join(jid_dir, MINIONS_P)] minions_cache.extend( glob.glob(os.path.join(jid_dir, SYNDIC_MINIONS_P.format('*'))) From 8a567f3d0b449ad072ab9738018c226c554e7317 Mon Sep 17 00:00:00 2001 From: Sam Yaple Date: Fri, 1 Dec 2017 15:17:48 -0500 Subject: [PATCH 135/159] Add new keystone module and states based on shade Test cases will be added in future patch --- salt/modules/keystoneng.py | 873 +++++++++++++++++++++++++++++ salt/states/keystone_domain.py | 122 ++++ salt/states/keystone_endpoint.py | 185 ++++++ salt/states/keystone_group.py | 140 +++++ salt/states/keystone_project.py | 141 +++++ salt/states/keystone_role.py | 106 ++++ salt/states/keystone_role_grant.py | 140 +++++ salt/states/keystone_service.py | 128 +++++ salt/states/keystone_user.py | 153 +++++ 9 files changed, 1988 insertions(+) create mode 100644 salt/modules/keystoneng.py create mode 100644 salt/states/keystone_domain.py create mode 100644 salt/states/keystone_endpoint.py create mode 100644 salt/states/keystone_group.py create mode 100644 salt/states/keystone_project.py create mode 100644 salt/states/keystone_role.py create mode 100644 salt/states/keystone_role_grant.py create mode 100644 salt/states/keystone_service.py create mode 100644 salt/states/keystone_user.py diff --git a/salt/modules/keystoneng.py b/salt/modules/keystoneng.py new file mode 100644 index 00000000000..d1601410f16 --- /dev/null +++ b/salt/modules/keystoneng.py @@ -0,0 +1,873 @@ +# -*- coding: utf-8 -*- +''' +Keystone module for interacting with OpenStack Keystone + +.. versionadded:: Nitrogen + +:depends:shade + +Example configuration + +.. code-block:: yaml + keystone: + cloud: default + +.. code-block:: yaml + keystone: + auth: + username: admin + password: password123 + user_domain_name: mydomain + project_name: myproject + project_domain_name: myproject + auth_url: https://example.org:5000/v3 + identity_api_version: 3 +''' + +from __future__ import absolute_import + +import salt.utils + +HAS_SHADE = False +try: + import shade + from shade.exc import OpenStackCloudException + HAS_SHADE = True +except ImportError: + pass + +__virtualname__ = 'keystoneng' + + +def __virtual__(): + ''' + Only load this module if shade python module is installed + ''' + if HAS_SHADE: + return __virtualname__ + return (False, 'The keystoneng execution module failed to load: shade python module is not available') + + +def compare_changes(obj, **kwargs): + ''' + Compare two dicts returning only keys that exist in the first dict and are + different in the second one + ''' + changes = {} + for k, v in obj.items(): + if k in kwargs: + if v != kwargs[k]: + changes[k] = kwargs[k] + return changes + + +def get_entity(ent_type, **kwargs): + ''' + Attempt to query Keystone for more information about an entity + ''' + try: + func = 'keystoneng.{}_get'.format(ent_type) + ent = __salt__[func](**kwargs) + except OpenStackCloudException as e: + # NOTE(SamYaple): If this error was something other than Forbidden we + # reraise the issue since we are not prepared to handle it + if 'HTTP 403' not in e.inner_exception[1][0]: + raise + + # NOTE(SamYaple): The user may be authorized to perform the function + # they are trying to do, but not authorized to search. In such a + # situation we want to trust that the user has passed a valid id, even + # though we cannot validate that this is a valid id + ent = kwargs['name'] + + return ent + + +def _clean_kwargs(keep_name=False, **kwargs): + ''' + Sanatize the the arguments for use with shade + ''' + if 'name' in kwargs and not keep_name: + kwargs['name_or_id'] = kwargs.pop('name') + + try: + clean_func = salt.utils.args.clean_kwargs + except AttributeError: + clean_func = salt.utils.clean_kwargs + return clean_func(**kwargs) + + +def setup_clouds(auth=None): + ''' + Call functions to create Shade cloud objects in __context__ to take + advantage of Shade's in-memory caching across several states + ''' + get_operator_cloud(auth) + get_openstack_cloud(auth) + + +def get_operator_cloud(auth=None): + ''' + Return an operator_cloud + ''' + if auth is None: + auth = __salt__['config.option']('keystone', {}) + if 'shade_opcloud' in __context__: + if __context__['shade_opcloud'].auth == auth: + return __context__['shade_opcloud'] + __context__['shade_opcloud'] = shade.operator_cloud(**auth) + return __context__['shade_opcloud'] + + +def get_openstack_cloud(auth=None): + ''' + Return an openstack_cloud + ''' + if auth is None: + auth = __salt__['config.option']('keystone', {}) + if 'shade_oscloud' in __context__: + if __context__['shade_oscloud'].auth == auth: + return __context__['shade_oscloud'] + __context__['shade_oscloud'] = shade.openstack_cloud(**auth) + return __context__['shade_oscloud'] + + +def group_create(auth=None, **kwargs): + ''' + Create a group + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.group_create name=group1 + salt '*' keystoneng.group_create name=group2 domain=domain1 description='my group2' + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(keep_name=True, **kwargs) + return cloud.create_group(**kwargs) + + +def group_delete(auth=None, **kwargs): + ''' + Delete a group + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.group_delete name=group1 + salt '*' keystoneng.group_delete name=group2 domain_id=b62e76fbeeff4e8fb77073f591cf211e + salt '*' keystoneng.group_delete name=0e4febc2a5ab4f2c8f374b054162506d + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.delete_group(**kwargs) + + +def group_update(auth=None, **kwargs): + ''' + Update a group + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.group_update name=group1 description='new description' + salt '*' keystoneng.group_create name=group2 domain_id=b62e76fbeeff4e8fb77073f591cf211e new_name=newgroupname + salt '*' keystoneng.group_create name=0e4febc2a5ab4f2c8f374b054162506d new_name=newgroupname + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + if 'new_name' in kwargs: + kwargs['name'] = kwargs.pop('new_name') + return cloud.update_group(**kwargs) + + +def group_list(auth=None, **kwargs): + ''' + List groups + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.group_list + salt '*' keystoneng.group_list domain_id=b62e76fbeeff4e8fb77073f591cf211e + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.list_groups(**kwargs) + + +def group_search(auth=None, **kwargs): + ''' + Search for groups + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.group_search name=group1 + salt '*' keystoneng.group_search domain_id=b62e76fbeeff4e8fb77073f591cf211e + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.search_groups(**kwargs) + + +def group_get(auth=None, **kwargs): + ''' + Get a single group + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.group_get name=group1 + salt '*' keystoneng.group_get name=group2 domain_id=b62e76fbeeff4e8fb77073f591cf211e + salt '*' keystoneng.group_get name=0e4febc2a5ab4f2c8f374b054162506d + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.get_group(**kwargs) + + +def project_create(auth=None, **kwargs): + ''' + Create a project + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.project_create name=project1 + salt '*' keystoneng.project_create name=project2 domain_id=b62e76fbeeff4e8fb77073f591cf211e + salt '*' keystoneng.project_create name=project3 enabled=False description='my project3' + ''' + cloud = get_openstack_cloud(auth) + kwargs = _clean_kwargs(keep_name=True, **kwargs) + return cloud.create_project(**kwargs) + + +def project_delete(auth=None, **kwargs): + ''' + Delete a project + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.project_delete name=project1 + salt '*' keystoneng.project_delete name=project2 domain_id=b62e76fbeeff4e8fb77073f591cf211e + salt '*' keystoneng.project_delete name=f315afcf12f24ad88c92b936c38f2d5a + ''' + cloud = get_openstack_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.delete_project(**kwargs) + + +def project_update(auth=None, **kwargs): + ''' + Update a project + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.project_update name=project1 new_name=newproject + salt '*' keystoneng.project_update name=project2 enabled=False description='new description' + ''' + cloud = get_openstack_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + if 'new_name' in kwargs: + kwargs['name'] = kwargs.pop('new_name') + return cloud.update_project(**kwargs) + + +def project_list(auth=None, **kwargs): + ''' + List projects + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.project_list + salt '*' keystoneng.project_list domain_id=b62e76fbeeff4e8fb77073f591cf211e + ''' + cloud = get_openstack_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.list_projects(**kwargs) + + +def project_search(auth=None, **kwargs): + ''' + Search projects + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.project_search + salt '*' keystoneng.project_search name=project1 + salt '*' keystoneng.project_search domain_id=b62e76fbeeff4e8fb77073f591cf211e + ''' + cloud = get_openstack_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.search_projects(**kwargs) + + +def project_get(auth=None, **kwargs): + ''' + Get a single project + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.project_get name=project1 + salt '*' keystoneng.project_get name=project2 domain_id=b62e76fbeeff4e8fb77073f591cf211e + salt '*' keystoneng.project_get name=f315afcf12f24ad88c92b936c38f2d5a + ''' + cloud = get_openstack_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.get_project(**kwargs) + + +def domain_create(auth=None, **kwargs): + ''' + Create a domain + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.domain_create name=domain1 + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(keep_name=True, **kwargs) + return cloud.create_domain(**kwargs) + + +def domain_delete(auth=None, **kwargs): + ''' + Delete a domain + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.domain_delete name=domain1 + salt '*' keystoneng.domain_delete name=b62e76fbeeff4e8fb77073f591cf211e + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.delete_domain(**kwargs) + + +def domain_update(auth=None, **kwargs): + ''' + Update a domain + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.domain_update name=domain1 new_name=newdomain + salt '*' keystoneng.domain_update name=domain1 enabled=True description='new description' + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + if 'new_name' in kwargs: + kwargs['name'] = kwargs.pop('new_name') + return cloud.update_domain(**kwargs) + + +def domain_list(auth=None, **kwargs): + ''' + List domains + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.domain_list + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.list_domains(**kwargs) + + +def domain_search(auth=None, **kwargs): + ''' + Search domains + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.domain_search + salt '*' keystoneng.domain_search name=domain1 + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.search_domains(**kwargs) + + +def domain_get(auth=None, **kwargs): + ''' + Get a single domain + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.domain_get name=domain1 + salt '*' keystoneng.domain_get name=b62e76fbeeff4e8fb77073f591cf211e + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.get_domain(**kwargs) + + +def role_create(auth=None, **kwargs): + ''' + Create a role + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.role_create name=role1 + salt '*' keystoneng.role_create name=role1 domain_id=b62e76fbeeff4e8fb77073f591cf211e + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(keep_name=True, **kwargs) + return cloud.create_role(**kwargs) + + +def role_delete(auth=None, **kwargs): + ''' + Delete a role + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.role_delete name=role1 domain_id=b62e76fbeeff4e8fb77073f591cf211e + salt '*' keystoneng.role_delete name=1eb6edd5525e4ac39af571adee673559 + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.delete_role(**kwargs) + + +def role_update(auth=None, **kwargs): + ''' + Update a role + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.role_update name=role1 new_name=newrole + salt '*' keystoneng.role_update name=1eb6edd5525e4ac39af571adee673559 new_name=newrole + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + if 'new_name' in kwargs: + kwargs['name'] = kwargs.pop('new_name') + return cloud.update_role(**kwargs) + + +def role_list(auth=None, **kwargs): + ''' + List roles + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.role_list + salt '*' keystoneng.role_list domain_id=b62e76fbeeff4e8fb77073f591cf211e + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.list_roles(**kwargs) + + +def role_search(auth=None, **kwargs): + ''' + Search roles + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.role_search + salt '*' keystoneng.role_search name=role1 + salt '*' keystoneng.role_search domain_id=b62e76fbeeff4e8fb77073f591cf211e + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.search_roles(**kwargs) + + +def role_get(auth=None, **kwargs): + ''' + Get a single role + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.role_get name=role1 + salt '*' keystoneng.role_get name=role1 domain_id=b62e76fbeeff4e8fb77073f591cf211e + salt '*' keystoneng.role_get name=1eb6edd5525e4ac39af571adee673559 + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.get_role(**kwargs) + + +def user_create(auth=None, **kwargs): + ''' + Create a user + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.user_create name=user1 + salt '*' keystoneng.user_create name=user2 password=1234 enabled=False + salt '*' keystoneng.user_create name=user3 domain_id=b62e76fbeeff4e8fb77073f591cf211e + ''' + cloud = get_openstack_cloud(auth) + kwargs = _clean_kwargs(keep_name=True, **kwargs) + return cloud.create_user(**kwargs) + + +def user_delete(auth=None, **kwargs): + ''' + Delete a user + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.user_delete name=user1 + salt '*' keystoneng.user_delete name=user2 domain_id=b62e76fbeeff4e8fb77073f591cf211e + salt '*' keystoneng.user_delete name=a42cbbfa1e894e839fd0f584d22e321f + ''' + cloud = get_openstack_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.delete_user(**kwargs) + + +def user_update(auth=None, **kwargs): + ''' + Update a user + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.user_update name=user1 enabled=False description='new description' + salt '*' keystoneng.user_update name=user1 new_name=newuser + ''' + cloud = get_openstack_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + if 'new_name' in kwargs: + kwargs['name'] = kwargs.pop('new_name') + return cloud.update_user(**kwargs) + + +def user_list(auth=None, **kwargs): + ''' + List users + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.user_list + salt '*' keystoneng.user_list domain_id=b62e76fbeeff4e8fb77073f591cf211e + ''' + cloud = get_openstack_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.list_users(**kwargs) + + +def user_search(auth=None, **kwargs): + ''' + List users + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.user_list + salt '*' keystoneng.user_list domain_id=b62e76fbeeff4e8fb77073f591cf211e + ''' + cloud = get_openstack_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.search_users(**kwargs) + + +def user_get(auth=None, **kwargs): + ''' + Get a single user + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.user_get name=user1 + salt '*' keystoneng.user_get name=user1 domain_id=b62e76fbeeff4e8fb77073f591cf211e + salt '*' keystoneng.user_get name=02cffaa173b2460f98e40eda3748dae5 + ''' + cloud = get_openstack_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.get_user(**kwargs) + + +def endpoint_create(auth=None, **kwargs): + ''' + Create an endpoint + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.endpoint_create interface=admin service=glance url=https://example.org:9292 + salt '*' keystoneng.endpoint_create interface=public service=glance region=RegionOne url=https://example.org:9292 + salt '*' keystoneng.endpoint_create interface=admin service=glance url=https://example.org:9292 enabled=True + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(keep_name=True, **kwargs) + return cloud.create_endpoint(**kwargs) + + +def endpoint_delete(auth=None, **kwargs): + ''' + Delete an endpoint + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.endpoint_delete id=3bee4bd8c2b040ee966adfda1f0bfca9 + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.delete_endpoint(**kwargs) + + +def endpoint_update(auth=None, **kwargs): + ''' + Update an endpoint + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.endpoint_update endpoint_id=4f961ad09d2d48948896bbe7c6a79717 interface=public enabled=False + salt '*' keystoneng.endpoint_update endpoint_id=4f961ad09d2d48948896bbe7c6a79717 region=newregion + salt '*' keystoneng.endpoint_update endpoint_id=4f961ad09d2d48948896bbe7c6a79717 service_name_or_id=glance url=https://example.org:9292 + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.update_endpoint(**kwargs) + + +def endpoint_list(auth=None, **kwargs): + ''' + List endpoints + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.endpoint_list + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.list_endpoints(**kwargs) + + +def endpoint_search(auth=None, **kwargs): + ''' + Search endpoints + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.endpoint_search + salt '*' keystoneng.endpoint_search id=02cffaa173b2460f98e40eda3748dae5 + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.search_endpoints(**kwargs) + + +def endpoint_get(auth=None, **kwargs): + ''' + Get a single endpoint + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.endpoint_get id=02cffaa173b2460f98e40eda3748dae5 + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.get_endpoint(**kwargs) + + +def service_create(auth=None, **kwargs): + ''' + Create a service + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.service_create name=glance type=image + salt '*' keystoneng.service_create name=glance type=image description="Image" + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(keep_name=True, **kwargs) + return cloud.create_service(**kwargs) + + +def service_delete(auth=None, **kwargs): + ''' + Delete a service + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.service_delete name=glance + salt '*' keystoneng.service_delete name=39cc1327cdf744ab815331554430e8ec + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.delete_service(**kwargs) + + +def service_update(auth=None, **kwargs): + ''' + Update a service + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.service_update name=cinder type=volumev2 + salt '*' keystoneng.service_update name=cinder description='new description' + salt '*' keystoneng.service_update name=ab4d35e269f147b3ae2d849f77f5c88f enabled=False + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.update_service(**kwargs) + + +def service_list(auth=None, **kwargs): + ''' + List services + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.service_list + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.list_services(**kwargs) + + +def service_search(auth=None, **kwargs): + ''' + Search services + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.service_search + salt '*' keystoneng.service_search name=glance + salt '*' keystoneng.service_search name=135f0403f8e544dc9008c6739ecda860 + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.search_services(**kwargs) + + +def service_get(auth=None, **kwargs): + ''' + Get a single service + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.service_get name=glance + salt '*' keystoneng.service_get name=75a5804638944b3ab54f7fbfcec2305a + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.get_service(**kwargs) + + +def role_assignment_list(auth=None, **kwargs): + ''' + List role assignments + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.role_assignment_list + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.list_role_assignments(**kwargs) + + +def role_grant(auth=None, **kwargs): + ''' + Grant a role in a project/domain to a user/group + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.role_grant name=role1 user=user1 project=project1 + salt '*' keystoneng.role_grant name=ddbe3e0ed74e4c7f8027bad4af03339d group=user1 project=project1 domain=domain1 + salt '*' keystoneng.role_grant name=ddbe3e0ed74e4c7f8027bad4af03339d group=19573afd5e4241d8b65c42215bae9704 project=1dcac318a83b4610b7a7f7ba01465548 + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.grant_role(**kwargs) + + +def role_revoke(auth=None, **kwargs): + ''' + Grant a role in a project/domain to a user/group + + CLI Example: + + .. code-block:: bash + + salt '*' keystoneng.role_revoke name=role1 user=user1 project=project1 + salt '*' keystoneng.role_revoke name=ddbe3e0ed74e4c7f8027bad4af03339d group=user1 project=project1 domain=domain1 + salt '*' keystoneng.role_revoke name=ddbe3e0ed74e4c7f8027bad4af03339d group=19573afd5e4241d8b65c42215bae9704 project=1dcac318a83b4610b7a7f7ba01465548 + ''' + cloud = get_operator_cloud(auth) + kwargs = _clean_kwargs(**kwargs) + return cloud.revoke_role(**kwargs) diff --git a/salt/states/keystone_domain.py b/salt/states/keystone_domain.py new file mode 100644 index 00000000000..2999e67f883 --- /dev/null +++ b/salt/states/keystone_domain.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- +''' +Management of OpenStack Keystone Domains +======================================== + +.. versionadded:: Nitrogen + +:depends: shade +:configuration: see :py:mod:`salt.modules.keystoneng` for setup instructions + +Example States + +.. code-block:: yaml + + create domain: + keystone_domain.present: + - name: domain1 + + create domain with optional params: + keystone_domain.present: + - name: domain1 + - enabled: False + - description: 'my domain' + + delete domain: + keystone_domain.absent: + - name: domain1 +''' + +from __future__ import absolute_import + +__virtualname__ = 'keystone_domain' + + +def __virtual__(): + if 'keystoneng.domain_get' in __salt__: + return __virtualname__ + return (False, 'The keystoneng execution module failed to load: shade python module is not available') + + +def present(name, auth=None, **kwargs): + ''' + Ensure domain exists and is up-to-date + + name + Name of the domain + + enabled + Boolean to control if domain is enabled + + description + An arbitrary description of the domain + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_clouds'](auth) + + domain = __salt__['keystoneng.domain_get'](name=name) + + if not domain: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = kwargs + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Domain {} will be created.'.format(name) + return ret + + kwargs['name'] = name + domain = __salt__['keystoneng.domain_create'](**kwargs) + ret['changes'] = domain + ret['comment'] = 'Created domain' + return ret + + changes = __salt__['keystoneng.compare_changes'](domain, **kwargs) + if changes: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = changes + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Domain {} will be updated.'.format(name) + return ret + + kwargs['domain_id'] = domain.id + __salt__['keystoneng.domain_update'](**kwargs) + ret['changes'].update(changes) + ret['comment'] = 'Updated domain' + + return ret + + +def absent(name, auth=None): + ''' + Ensure domain does not exist + + name + Name of the domain + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_clouds'](auth) + + domain = __salt__['keystoneng.domain_get'](name=name) + + if domain: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = {'name': name} + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Domain {} will be deleted.'.format(name) + return ret + + __salt__['keystoneng.domain_delete'](name=domain) + ret['changes']['id'] = domain.id + ret['comment'] = 'Deleted domain' + + return ret diff --git a/salt/states/keystone_endpoint.py b/salt/states/keystone_endpoint.py new file mode 100644 index 00000000000..759fe8c2e19 --- /dev/null +++ b/salt/states/keystone_endpoint.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- +''' +Management of OpenStack Keystone Endpoints +========================================== + +.. versionadded:: Nitrogen + +:depends: shade +:configuration: see :py:mod:`salt.modules.keystoneng` for setup instructions + +Example States + +.. code-block:: yaml + + create endpoint: + keystone_endpoint.present: + - name: public + - url: https://example.org:9292 + - region: RegionOne + - service_name: glance + + destroy endpoint: + keystone_endpoint.absent: + - name: public + - url: https://example.org:9292 + - region: RegionOne + - service_name: glance + + create multiple endpoints: + keystone_endpoint.absent: + - names: + - public + - admin + - internal + - url: https://example.org:9292 + - region: RegionOne + - service_name: glance +''' + +from __future__ import absolute_import + +__virtualname__ = 'keystone_endpoint' + + +def __virtual__(): + if 'keystoneng.endpoint_get' in __salt__: + return __virtualname__ + return (False, 'The keystoneng execution module failed to load: shade python module is not available') + + +def _common(ret, name, service_name, kwargs): + ''' + Returns: tuple whose first element is a bool indicating success or failure + and the second element is either a ret dict for salt or an object + ''' + if 'interface' not in kwargs and 'public_url' not in kwargs: + kwargs['interface'] = name + service = __salt__['keystoneng.service_get'](name_or_id=service_name) + + if not service: + ret['comment'] = 'Cannot find service' + ret['result'] = False + return (False, ret) + + filters = kwargs.copy() + filters.pop('enabled', None) + filters.pop('url', None) + filters['service_id'] = service.id + kwargs['service_name_or_id'] = service.id + endpoints = __salt__['keystoneng.endpoint_search'](filters=filters) + + if len(endpoints) > 1: + ret['comment'] = "Multiple endpoints match criteria" + ret['result'] = False + return ret + endpoint = endpoints[0] if endpoints else None + return (True, endpoint) + + +def present(name, service_name, auth=None, **kwargs): + ''' + Ensure an endpoint exists and is up-to-date + + name + Interface name + + url + URL of the endpoint + + service_name + Service name or ID + + region + The region name to assign the endpoint + + enabled + Boolean to control if endpoint is enabled + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_clouds'](auth) + + success, val = _, endpoint = _common(ret, name, service_name, kwargs) + if not success: + return val + + if not endpoint: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = kwargs + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Endpoint will be created.' + return ret + + # NOTE(SamYaple): Endpoints are returned as a list which can contain + # several items depending on the options passed + endpoints = __salt__['keystoneng.endpoint_create'](**kwargs) + if len(endpoints) == 1: + ret['changes'] = endpoints[0] + else: + for i, endpoint in enumerate(endpoints): + ret['changes'][i] = endpoint + ret['comment'] = 'Created endpoint' + return ret + + changes = __salt__['keystoneng.compare_changes'](endpoint, **kwargs) + if changes: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = changes + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Endpoint will be updated.' + return ret + + kwargs['endpoint_id'] = endpoint.id + __salt__['keystoneng.endpoint_update'](**kwargs) + ret['changes'].update(changes) + ret['comment'] = 'Updated endpoint' + + return ret + + +def absent(name, service_name, auth=None, **kwargs): + ''' + Ensure an endpoint does not exists + + name + Interface name + + url + URL of the endpoint + + service_name + Service name or ID + + region + The region name to assign the endpoint + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_clouds'](auth) + + success, val = _, endpoint = _common(ret, name, service_name, kwargs) + if not success: + return val + + if endpoint: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = {'id': endpoint.id} + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Endpoint will be deleted.' + return ret + + __salt__['keystoneng.endpoint_delete'](id=endpoint.id) + ret['changes']['id'] = endpoint.id + ret['comment'] = 'Deleted endpoint' + + return ret diff --git a/salt/states/keystone_group.py b/salt/states/keystone_group.py new file mode 100644 index 00000000000..64c46eb6fa5 --- /dev/null +++ b/salt/states/keystone_group.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +''' +Management of OpenStack Keystone Groups +======================================= + +.. versionadded:: Nitrogen + +:depends: shade +:configuration: see :py:mod:`salt.modules.keystoneng` for setup instructions + +Example States + +.. code-block:: yaml + + create group: + keystone_group.present: + - name: group1 + + delete group: + keystone_group.absent: + - name: group1 + + create group with optional params: + keystone_group.present: + - name: group1 + - domain: domain1 + - description: 'my group' +''' + +from __future__ import absolute_import + +__virtualname__ = 'keystone_endpoint' + + +def __virtual__(): + if 'keystoneng.group_get' in __salt__: + return __virtualname__ + return (False, 'The keystoneng execution module failed to load: shade python module is not available') + + +def _common(kwargs): + ''' + Returns: None if group wasn't found, otherwise a group object + ''' + search_kwargs = {'name': kwargs['name']} + if 'domain' in kwargs: + domain = __salt__['keystoneng.get_entity']( + 'domain', name=kwargs.pop('domain')) + domain_id = domain.id if hasattr(domain, 'id') else domain + search_kwargs['filters'] = {'domain_id': domain_id} + kwargs['domain'] = domain + + return __salt__['keystoneng.group_get'](**search_kwargs) + + +def present(name, auth=None, **kwargs): + ''' + Ensure an group exists and is up-to-date + + name + Name of the group + + domain + The name or id of the domain + + description + An arbitrary description of the group + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_cloud'](auth) + + kwargs['name'] = name + group = _common(kwargs) + + if group is None: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = kwargs + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Group will be created.' + return ret + + group = __salt__['keystoneng.group_create'](**kwargs) + ret['changes'] = group + ret['comment'] = 'Created group' + return ret + + changes = __salt__['keystoneng.compare_changes'](group, **kwargs) + if changes: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = changes + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Group will be updated.' + return ret + + __salt__['keystoneng.group_update'](**kwargs) + ret['changes'].update(changes) + ret['comment'] = 'Updated group' + + return ret + + +def absent(name, auth=None, **kwargs): + ''' + Ensure group does not exist + + name + Name of the group + + domain + The name or id of the domain + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_cloud'](auth) + + kwargs['name'] = name + group = _common(kwargs) + + if group: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = {'id': group.id} + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Group will be deleted.' + return ret + + __salt__['keystoneng.group_delete'](name=group) + ret['changes']['id'] = group.id + ret['comment'] = 'Deleted group' + + return ret diff --git a/salt/states/keystone_project.py b/salt/states/keystone_project.py new file mode 100644 index 00000000000..167d722194e --- /dev/null +++ b/salt/states/keystone_project.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- +''' +Management of OpenStack Keystone Projects +========================================= + +.. versionadded:: Nitrogen + +:depends: shade +:configuration: see :py:mod:`salt.modules.keystoneng` for setup instructions + +Example States + +.. code-block:: yaml + + create project: + keystone_project.present: + - name: project1 + + delete project: + keystone_project.absent: + - name: project1 + + create project with optional params: + keystone_project.present: + - name: project1 + - domain: domain1 + - enabled: False + - description: 'my project' +''' + +from __future__ import absolute_import + +__virtualname__ = 'keystone_project' + + +def __virtual__(): + if 'keystoneng.project_get' in __salt__: + return __virtualname__ + return (False, 'The keystoneng execution module failed to load: shade python module is not available') + + +def _common(name, kwargs): + ''' + Returns: None if project wasn't found, otherwise a group object + ''' + search_kwargs = {'name': name} + if 'domain' in kwargs: + domain = __salt__['keystoneng.get_entity']( + 'domain', name=kwargs.pop('domain')) + domain_id = domain.id if hasattr(domain, 'id') else domain + search_kwargs['domain_id'] = domain_id + kwargs['domain_id'] = domain_id + + return __salt__['keystoneng.project_get'](**search_kwargs) + + +def present(name, auth=None, **kwargs): + ''' + Ensure a project exists and is up-to-date + + name + Name of the project + + domain + The name or id of the domain + + description + An arbitrary description of the project + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_clouds'](auth) + + kwargs['name'] = name + project = _common(name, kwargs) + + if project is None: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = kwargs + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Project will be created.' + return ret + + project = __salt__['keystoneng.project_create'](**kwargs) + ret['changes'] = project + ret['comment'] = 'Created project' + return ret + + changes = __salt__['keystoneng.compare_changes'](project, **kwargs) + if changes: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = changes + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Project will be updated.' + return ret + + __salt__['keystoneng.project_update'](**kwargs) + ret['changes'].update(changes) + ret['comment'] = 'Updated project' + + return ret + + +def absent(name, auth=None, **kwargs): + ''' + Ensure a project does not exists + + name + Name of the project + + domain + The name or id of the domain + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_clouds'](auth) + + kwargs['name'] = name + project = _common(name, kwargs) + + if project: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = {'id': project.id} + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Project will be deleted.' + return ret + + __salt__['keystoneng.project_delete'](name=project) + ret['changes']['id'] = project.id + ret['comment'] = 'Deleted project' + + return ret diff --git a/salt/states/keystone_role.py b/salt/states/keystone_role.py new file mode 100644 index 00000000000..9c067b8cd20 --- /dev/null +++ b/salt/states/keystone_role.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +''' +Management of OpenStack Keystone Roles +====================================== + +.. versionadded:: Nitrogen + +:depends: shade +:configuration: see :py:mod:`salt.modules.keystoneng` for setup instructions + +Example States + +.. code-block:: yaml + + create role: + keystone_role.present: + - name: role1 + + delete role: + keystone_role.absent: + - name: role1 + + create role with optional params: + keystone_role.present: + - name: role1 + - description: 'my group' +''' + +from __future__ import absolute_import + +__virtualname__ = 'keystone_role' + + +def __virtual__(): + if 'keystoneng.role_get' in __salt__: + return __virtualname__ + return (False, 'The keystoneng execution module failed to load: shade python module is not available') + + +def present(name, auth=None, **kwargs): + ''' + Ensure an role exists + + name + Name of the role + + description + An arbitrary description of the role + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_clouds'](auth) + + kwargs['name'] = name + role = __salt__['keystoneng.role_get'](**kwargs) + + if not role: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = kwargs + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Role will be created.' + return ret + + role = __salt__['keystoneng.role_create'](**kwargs) + ret['changes']['id'] = role.id + ret['changes']['name'] = role.name + ret['comment'] = 'Created role' + return ret + # NOTE(SamYaple): Update support pending https://review.openstack.org/#/c/496992/ + return ret + + +def absent(name, auth=None, **kwargs): + ''' + Ensure role does not exist + + name + Name of the role + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_clouds'](auth) + + kwargs['name'] = name + role = __salt__['keystoneng.role_get'](**kwargs) + + if role: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = {'id': role.id} + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Role will be deleted.' + return ret + + __salt__['keystoneng.role_delete'](name=role) + ret['changes']['id'] = role.id + ret['comment'] = 'Deleted role' + + return ret diff --git a/salt/states/keystone_role_grant.py b/salt/states/keystone_role_grant.py new file mode 100644 index 00000000000..37a0bb0b3dc --- /dev/null +++ b/salt/states/keystone_role_grant.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +''' +Management of OpenStack Keystone Role Grants +============================================ + +.. versionadded:: Nitrogen + +:depends: shade +:configuration: see :py:mod:`salt.modules.keystoneng` for setup instructions + +Example States + +.. code-block:: yaml + + create group: + keystone_group.present: + - name: group1 + + delete group: + keystone_group.absent: + - name: group1 + + create group with optional params: + keystone_group.present: + - name: group1 + - domain: domain1 + - description: 'my group' +''' + +from __future__ import absolute_import + +__virtualname__ = 'keystone_role_grant' + + +def __virtual__(): + if 'keystoneng.role_grant' in __salt__: + return __virtualname__ + return (False, 'The keystoneng execution module failed to load: shade python module is not available') + + +def _get_filters(kwargs): + role_kwargs = {'name': kwargs.pop('role')} + if 'role_domain' in kwargs: + domain = __salt__['keystoneng.get_entity']( + 'domain', name=kwargs.pop('role_domain')) + if domain: + role_kwargs['domain_id'] = domain.id \ + if hasattr(domain, 'id') else domain + role = __salt__['keystoneng.role_get'](**role_kwargs) + kwargs['name'] = role + filters = {'role': role.id if hasattr(role, 'id') else role} + + if 'domain' in kwargs: + domain = __salt__['keystoneng.get_entity']( + 'domain', name=kwargs.pop('domain')) + kwargs['domain'] = filters['domain'] = \ + domain.id if hasattr(domain, 'id') else domain + + if 'project' in kwargs: + project_kwargs = {'name': kwargs.pop('project')} + if 'project_domain' in kwargs: + domain = __salt__['keystoneng.get_entity']( + 'domain', name=kwargs.pop('project_domain')) + if domain: + project_kwargs['domain_id'] = domain.id + project = __salt__['keystoneng.get_entity']( + 'project', **project_kwargs) + kwargs['project'] = project + filters['project'] = project.id if hasattr(project, 'id') else project + + if 'user' in kwargs: + user_kwargs = {'name': kwargs.pop('user')} + if 'user_domain' in kwargs: + domain = __salt__['keystoneng.get_entity']( + 'domain', name=kwargs.pop('user_domain')) + if domain: + user_kwargs['domain_id'] = domain.id + user = __salt__['keystoneng.get_entity']('user', **user_kwargs) + kwargs['user'] = user + filters['user'] = user.id if hasattr(user, 'id') else user + + if 'group' in kwargs: + group_kwargs = {'name': kwargs['group']} + if 'group_domain' in kwargs: + domain = __salt__['keystoneng.get_entity']( + 'domain', name=kwargs.pop('group_domain')) + if domain: + group_kwargs['domain_id'] = domain.id + group = __salt__['keystoneng.get_entity']('group', **group_kwargs) + + kwargs['group'] = group + filters['group'] = group.id if hasattr(group, 'id') else group + + return filters, kwargs + + +def present(name, auth=None, **kwargs): + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_clouds'](auth) + + if 'role' not in kwargs: + kwargs['role'] = name + filters, kwargs = _get_filters(kwargs) + + grants = __salt__['keystoneng.role_assignment_list'](filters=filters) + + if not grants: + __salt__['keystoneng.role_grant'](**kwargs) + for k, v in filters.items(): + ret['changes'][k] = v + ret['comment'] = 'Granted role assignment' + + return ret + + +def absent(name, auth=None, **kwargs): + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_clouds'](auth) + + if 'role' not in kwargs: + kwargs['role'] = name + filters, kwargs = _get_filters(kwargs) + + grants = __salt__['keystoneng.role_assignment_list'](filters=filters) + + if grants: + __salt__['keystoneng.role_revoke'](**kwargs) + for k, v in filters.items(): + ret['changes'][k] = v + ret['comment'] = 'Revoked role assignment' + + return ret diff --git a/salt/states/keystone_service.py b/salt/states/keystone_service.py new file mode 100644 index 00000000000..c3d9c404d55 --- /dev/null +++ b/salt/states/keystone_service.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- +''' +Management of OpenStack Keystone Services +========================================= + +.. versionadded:: Nitrogen + +:depends: shade +:configuration: see :py:mod:`salt.modules.keystoneng` for setup instructions + +Example States + +.. code-block:: yaml + + create service: + keystone_service.present: + - name: glance + - type: image + + delete service: + keystone_service.absent: + - name: glance + + create service with optional params: + keystone_service.present: + - name: glance + - type: image + - enabled: False + - description: 'OpenStack Image' +''' + +from __future__ import absolute_import + +__virtualname__ = 'keystone_service' + + +def __virtual__(): + if 'keystoneng.service_get' in __salt__: + return __virtualname__ + return (False, 'The keystoneng execution module failed to load: shade python module is not available') + + +def present(name, auth=None, **kwargs): + ''' + Ensure an service exists and is up-to-date + + name + Name of the group + + type + Service type + + enabled + Boolean to control if service is enabled + + description + An arbitrary description of the service + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_clouds'](auth) + + service = __salt__['keystoneng.service_get'](name=name) + + if service is None: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = kwargs + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Service will be created.' + return ret + + kwargs['name'] = name + service = __salt__['keystoneng.service_create'](**kwargs) + ret['changes'] = service + ret['comment'] = 'Created service' + return ret + + changes = __salt__['keystoneng.compare_changes'](service, **kwargs) + if changes: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = changes + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Service will be updated.' + return ret + + kwargs['name'] = service + __salt__['keystoneng.service_update'](**kwargs) + ret['changes'].update(changes) + ret['comment'] = 'Updated service' + + return ret + + +def absent(name, auth=None): + ''' + Ensure service does not exist + + name + Name of the service + ''' + + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_clouds'](auth) + + service = __salt__['keystoneng.service_get'](name=name) + + if service: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = {'id': service.id} + ret['pchanges'] = ret['changes'] + ret['comment'] = 'Service will be deleted.' + return ret + + __salt__['keystoneng.service_delete'](name=service) + ret['changes']['id'] = service.id + ret['comment'] = 'Deleted service' + + return ret diff --git a/salt/states/keystone_user.py b/salt/states/keystone_user.py new file mode 100644 index 00000000000..a2a40c902c6 --- /dev/null +++ b/salt/states/keystone_user.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +''' +Management of OpenStack Keystone Users +====================================== + +.. versionadded:: Nitrogen + +:depends: shade +:configuration: see :py:mod:`salt.modules.keystoneng` for setup instructions + +Example States + +.. code-block:: yaml + + create user: + keystone_user.present: + - name: user1 + + delete user: + keystone_user.absent: + - name: user1 + + create user with optional params: + keystone_user.present: + - name: user1 + - domain: domain1 + - enabled: False + - password: password123 + - email: "user1@example.org" + - description: 'my user' +''' + +from __future__ import absolute_import + +__virtualname__ = 'keystone_user' + + +def __virtual__(): + if 'keystoneng.user_get' in __salt__: + return __virtualname__ + return (False, 'The keystoneng execution module failed to load: shade python module is not available') + + +def _common(kwargs): + ''' + Returns: None if user wasn't found, otherwise a user object + ''' + search_kwargs = {'name': kwargs['name']} + if 'domain' in kwargs: + domain = __salt__['keystoneng.get_entity']( + 'domain', name=kwargs.pop('domain')) + domain_id = domain.id if hasattr(domain, 'id') else domain + search_kwargs['domain_id'] = domain_id + kwargs['domain_id'] = domain_id + + return __salt__['keystoneng.user_get'](**search_kwargs) + + +def present(name, auth=None, **kwargs): + ''' + Ensure domain exists and is up-to-date + + name + Name of the domain + + domain + The name or id of the domain + + enabled + Boolean to control if domain is enabled + + description + An arbitrary description of the domain + + password + The user password + + email + The users email address + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_clouds'](auth) + + kwargs['name'] = name + user = _common(kwargs) + + if user is None: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = kwargs + ret['pchanges'] = ret['changes'] + ret['comment'] = 'User will be created.' + return ret + + user = __salt__['keystoneng.user_create'](**kwargs) + ret['changes'] = user + ret['comment'] = 'Created user' + return ret + + changes = __salt__['keystoneng.compare_changes'](user, **kwargs) + if changes: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = changes + ret['pchanges'] = ret['changes'] + ret['comment'] = 'User will be updated.' + return ret + + kwargs['name'] = user + __salt__['keystoneng.user_update'](**kwargs) + ret['changes'].update(changes) + ret['comment'] = 'Updated user' + + return ret + + +def absent(name, auth=None, **kwargs): + ''' + Ensure user does not exists + + name + Name of the user + + domain + The name or id of the domain + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + __salt__['keystoneng.setup_clouds'](auth) + + kwargs['name'] = name + user = _common(kwargs) + + if user: + if __opts__['test'] is True: + ret['result'] = None + ret['changes'] = {'id': user.id} + ret['pchanges'] = ret['changes'] + ret['comment'] = 'User will be deleted.' + return ret + + __salt__['keystoneng.user_delete'](name=user) + ret['changes']['id'] = user.id + ret['comment'] = 'Deleted user' + + return ret From bf9dc026bb2744dbc66cf4ed3620d254677a7ad1 Mon Sep 17 00:00:00 2001 From: Eric Radman Date: Fri, 1 Dec 2017 11:46:30 -0500 Subject: [PATCH 136/159] Update salt_boostrap command line help summary - Use exact indentation produced by the bootstrap-salt command - normal package installs work fine on OpenBSD --- doc/topics/tutorials/salt_bootstrap.rst | 160 ++++++++++++++++-------- 1 file changed, 106 insertions(+), 54 deletions(-) diff --git a/doc/topics/tutorials/salt_bootstrap.rst b/doc/topics/tutorials/salt_bootstrap.rst index 93b5dc3b2d5..4b11ee91dab 100644 --- a/doc/topics/tutorials/salt_bootstrap.rst +++ b/doc/topics/tutorials/salt_bootstrap.rst @@ -78,7 +78,7 @@ UNIX systems **BSD**: -- OpenBSD (``pip`` installation) +- OpenBSD - FreeBSD 9/10/11 **SunOS**: @@ -272,66 +272,118 @@ Here's a summary of the command line options: $ sh bootstrap-salt.sh -h - Usage : bootstrap-salt.sh [options] - Installation types: - - stable (default) - - stable [version] (ubuntu specific) - - daily (ubuntu specific) - - testing (redhat specific) - - git + - stable Install latest stable release. This is the default + install type + - stable [branch] Install latest version on a branch. Only supported + for packages available at repo.saltstack.com + - stable [version] Install a specific version. Only supported for + packages available at repo.saltstack.com + - daily Ubuntu specific: configure SaltStack Daily PPA + - testing RHEL-family specific: configure EPEL testing repo + - git Install from the head of the develop branch + - git [ref] Install from any git ref (such as a branch, tag, or + commit) Examples: - bootstrap-salt.sh - bootstrap-salt.sh stable - - bootstrap-salt.sh stable 2014.7 + - bootstrap-salt.sh stable 2017.7 + - bootstrap-salt.sh stable 2017.7.2 - bootstrap-salt.sh daily - bootstrap-salt.sh testing - bootstrap-salt.sh git - - bootstrap-salt.sh git develop - - bootstrap-salt.sh git v0.17.0 - - bootstrap-salt.sh git 8c3fadf15ec183e5ce8c63739850d543617e4357 + - bootstrap-salt.sh git 2017.7 + - bootstrap-salt.sh git v2017.7.2 + - bootstrap-salt.sh git 06f249901a2e2f1ed310d58ea3921a129f214358 Options: - -h Display this message - -v Display script version - -n No colours. - -D Show debug output. - -c Temporary configuration directory - -g Salt repository URL. (default: git://github.com/saltstack/salt.git) - -G Instead of cloning from git://github.com/saltstack/salt.git, clone from https://github.com/saltstack/salt.git (Usually necessary on systems which have the regular git protocol port blocked, where https usually is not) - -k Temporary directory holding the minion keys which will pre-seed - the master. - -s Sleep time used when waiting for daemons to start, restart and when checking - for the services running. Default: 3 - -M Also install salt-master - -S Also install salt-syndic - -N Do not install salt-minion - -X Do not start daemons after installation - -C Only run the configuration function. This option automatically - bypasses any installation. - -P Allow pip based installations. On some distributions the required salt - packages or its dependencies are not available as a package for that - distribution. Using this flag allows the script to use pip as a last - resort method. NOTE: This only works for functions which actually - implement pip based installations. - -F Allow copied files to overwrite existing(config, init.d, etc) - -U If set, fully upgrade the system prior to bootstrapping salt - -K If set, keep the temporary files in the temporary directories specified - with -c and -k. - -I If set, allow insecure connections while downloading any files. For - example, pass '--no-check-certificate' to 'wget' or '--insecure' to 'curl' - -A Pass the salt-master DNS name or IP. This will be stored under - ${BS_SALT_ETC_DIR}/minion.d/99-master-address.conf - -i Pass the salt-minion id. This will be stored under - ${BS_SALT_ETC_DIR}/minion_id - -L Install the Apache Libcloud package if possible(required for salt-cloud) - -p Extra-package to install while installing salt dependencies. One package - per -p flag. You're responsible for providing the proper package name. - -d Disable check_service functions. Setting this flag disables the - 'install__check_services' checks. You can also do this by - touching /tmp/disable_salt_checks on the target host. Defaults ${BS_FALSE} - -H Use the specified http proxy for the installation - -Z Enable external software source for newer ZeroMQ(Only available for RHEL/CentOS/Fedora/Ubuntu based distributions) - -b Assume that dependencies are already installed and software sources are set up. - If git is selected, git tree is still checked out as dependency step. + -h Display this message + -v Display script version + -n No colours + -D Show debug output + -c Temporary configuration directory + -g Salt Git repository URL. Default: https://github.com/saltstack/salt.git + -w Install packages from downstream package repository rather than + upstream, saltstack package repository. This is currently only + implemented for SUSE. + -k Temporary directory holding the minion keys which will pre-seed + the master. + -s Sleep time used when waiting for daemons to start, restart and when + checking for the services running. Default: 3 + -L Also install salt-cloud and required python-libcloud package + -M Also install salt-master + -S Also install salt-syndic + -N Do not install salt-minion + -X Do not start daemons after installation + -d Disables checking if Salt services are enabled to start on system boot. + You can also do this by touching /tmp/disable_salt_checks on the target + host. Default: ${BS_FALSE} + -P Allow pip based installations. On some distributions the required salt + packages or its dependencies are not available as a package for that + distribution. Using this flag allows the script to use pip as a last + resort method. NOTE: This only works for functions which actually + implement pip based installations. + -U If set, fully upgrade the system prior to bootstrapping Salt + -I If set, allow insecure connections while downloading any files. For + example, pass '--no-check-certificate' to 'wget' or '--insecure' to + 'curl'. On Debian and Ubuntu, using this option with -U allows to obtain + GnuPG archive keys insecurely if distro has changed release signatures. + -F Allow copied files to overwrite existing (config, init.d, etc) + -K If set, keep the temporary files in the temporary directories specified + with -c and -k + -C Only run the configuration function. Implies -F (forced overwrite). + To overwrite Master or Syndic configs, -M or -S, respectively, must + also be specified. Salt installation will be ommitted, but some of the + dependencies could be installed to write configuration with -j or -J. + -A Pass the salt-master DNS name or IP. This will be stored under + ${BS_SALT_ETC_DIR}/minion.d/99-master-address.conf + -i Pass the salt-minion id. This will be stored under + ${BS_SALT_ETC_DIR}/minion_id + -p Extra-package to install while installing Salt dependencies. One package + per -p flag. You're responsible for providing the proper package name. + -H Use the specified HTTP proxy for all download URLs (including https://). + For example: http://myproxy.example.com:3128 + -Z Enable additional package repository for newer ZeroMQ + (only available for RHEL/CentOS/Fedora/Ubuntu based distributions) + -b Assume that dependencies are already installed and software sources are + set up. If git is selected, git tree is still checked out as dependency + step. + -f Force shallow cloning for git installations. + This may result in an "n/a" in the version number. + -l Disable ssl checks. When passed, switches "https" calls to "http" where + possible. + -V Install Salt into virtualenv + (only available for Ubuntu based distributions) + -a Pip install all Python pkg dependencies for Salt. Requires -V to install + all pip pkgs into the virtualenv. + (Only available for Ubuntu based distributions) + -r Disable all repository configuration performed by this script. This + option assumes all necessary repository configuration is already present + on the system. + -R Specify a custom repository URL. Assumes the custom repository URL + points to a repository that mirrors Salt packages located at + repo.saltstack.com. The option passed with -R replaces the + "repo.saltstack.com". If -R is passed, -r is also set. Currently only + works on CentOS/RHEL and Debian based distributions. + -J Replace the Master config file with data passed in as a JSON string. If + a Master config file is found, a reasonable effort will be made to save + the file with a ".bak" extension. If used in conjunction with -C or -F, + no ".bak" file will be created as either of those options will force + a complete overwrite of the file. + -j Replace the Minion config file with data passed in as a JSON string. If + a Minion config file is found, a reasonable effort will be made to save + the file with a ".bak" extension. If used in conjunction with -C or -F, + no ".bak" file will be created as either of those options will force + a complete overwrite of the file. + -q Quiet salt installation from git (setup.py install -q) + -x Changes the python version used to install a git version of salt. Currently + this is considered experimental and has only been tested on Centos 6. This + only works for git installations. + -y Installs a different python version on host. Currently this has only been + tested with Centos 6 and is considered experimental. This will install the + ius repo on the box if disable repo is false. This must be used in conjunction + with -x . For example: + sh bootstrap.sh -P -y -x python2.7 git v2016.11.3 + The above will install python27 and install the git version of salt using the + python2.7 executable. This only works for git and pip installations. From 371e2102470a7f7575fdec381097f60fcdb5cadb Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 1 Dec 2017 12:34:57 -0700 Subject: [PATCH 137/159] Add ability to reset file system object perms --- salt/modules/file.py | 54 ++-- salt/modules/win_file.py | 436 ++++++++++++++++++++++----------- salt/states/file.py | 139 +++++++++-- tests/unit/states/test_file.py | 7 +- 4 files changed, 450 insertions(+), 186 deletions(-) diff --git a/salt/modules/file.py b/salt/modules/file.py index f7145d06775..ecbd5912eca 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -5203,13 +5203,18 @@ def manage_file(name, 'Replace symbolic link with regular file' if salt.utils.platform.is_windows(): - ret = check_perms(name, - ret, - kwargs.get('win_owner'), - kwargs.get('win_perms'), - kwargs.get('win_deny_perms'), - None, - kwargs.get('win_inheritance')) + # This function resides in win_file.py and will be available + # on Windows. The local function will be overridden + # pylint: disable=E1120,E1121,E1123 + ret = check_perms( + path=name, + ret=ret, + owner=kwargs.get('win_owner'), + grant_perms=kwargs.get('win_perms'), + deny_perms=kwargs.get('win_deny_perms'), + inheritance=kwargs.get('win_inheritance', True), + reset=kwargs.get('win_perms_reset', False)) + # pylint: enable=E1120,E1121,E1123 else: ret, _ = check_perms(name, ret, user, group, mode, attrs, follow_symlinks) @@ -5250,13 +5255,15 @@ def manage_file(name, if salt.utils.platform.is_windows(): # This function resides in win_file.py and will be available # on Windows. The local function will be overridden - # pylint: disable=E1121 - makedirs_(name, - kwargs.get('win_owner'), - kwargs.get('win_perms'), - kwargs.get('win_deny_perms'), - kwargs.get('win_inheritance')) - # pylint: enable=E1121 + # pylint: disable=E1120,E1121,E1123 + makedirs_( + path=name, + owner=kwargs.get('win_owner'), + grant_perms=kwargs.get('win_perms'), + deny_perms=kwargs.get('win_deny_perms'), + inheritance=kwargs.get('win_inheritance', True), + reset=kwargs.get('win_perms_reset', False)) + # pylint: enable=E1120,E1121,E1123 else: makedirs_(name, user=user, group=group, mode=dir_mode) @@ -5369,13 +5376,18 @@ def manage_file(name, mode = oct((0o777 ^ mask) & 0o666) if salt.utils.platform.is_windows(): - ret = check_perms(name, - ret, - kwargs.get('win_owner'), - kwargs.get('win_perms'), - kwargs.get('win_deny_perms'), - None, - kwargs.get('win_inheritance')) + # This function resides in win_file.py and will be available + # on Windows. The local function will be overridden + # pylint: disable=E1120,E1121,E1123 + ret = check_perms( + path=name, + ret=ret, + owner=kwargs.get('win_owner'), + grant_perms=kwargs.get('win_perms'), + deny_perms=kwargs.get('win_deny_perms'), + inheritance=kwargs.get('win_inheritance', True), + reset=kwargs.get('win_perms_reset', False)) + # pylint: enable=E1120,E1121,E1123 else: ret, _ = check_perms(name, ret, user, group, mode, attrs) diff --git a/salt/modules/win_file.py b/salt/modules/win_file.py index f1b55fafad4..372dabbb56c 100644 --- a/salt/modules/win_file.py +++ b/salt/modules/win_file.py @@ -1218,41 +1218,55 @@ def mkdir(path, owner=None, grant_perms=None, deny_perms=None, - inheritance=True): + inheritance=True, + reset=False): ''' Ensure that the directory is available and permissions are set. Args: - path (str): The full path to the directory. + path (str): + The full path to the directory. - owner (str): The owner of the directory. If not passed, it will be the - account that created the directory, likely SYSTEM + owner (str): + The owner of the directory. If not passed, it will be the account + that created the directory, likely SYSTEM - grant_perms (dict): A dictionary containing the user/group and the basic - permissions to grant, ie: ``{'user': {'perms': 'basic_permission'}}``. - You can also set the ``applies_to`` setting here. The default is - ``this_folder_subfolders_files``. Specify another ``applies_to`` setting - like this: + grant_perms (dict): + A dictionary containing the user/group and the basic permissions to + grant, ie: ``{'user': {'perms': 'basic_permission'}}``. You can also + set the ``applies_to`` setting here. The default is + ``this_folder_subfolders_files``. Specify another ``applies_to`` + setting like this: - .. code-block:: yaml + .. code-block:: yaml - {'user': {'perms': 'full_control', 'applies_to': 'this_folder'}} + {'user': {'perms': 'full_control', 'applies_to': 'this_folder'}} - To set advanced permissions use a list for the ``perms`` parameter, ie: + To set advanced permissions use a list for the ``perms`` parameter, + ie: - .. code-block:: yaml + .. code-block:: yaml - {'user': {'perms': ['read_attributes', 'read_ea'], 'applies_to': 'this_folder'}} + {'user': {'perms': ['read_attributes', 'read_ea'], 'applies_to': 'this_folder'}} - deny_perms (dict): A dictionary containing the user/group and - permissions to deny along with the ``applies_to`` setting. Use the same - format used for the ``grant_perms`` parameter. Remember, deny - permissions supersede grant permissions. + deny_perms (dict): + A dictionary containing the user/group and permissions to deny along + with the ``applies_to`` setting. Use the same format used for the + ``grant_perms`` parameter. Remember, deny permissions supersede + grant permissions. - inheritance (bool): If True the object will inherit permissions from the - parent, if False, inheritance will be disabled. Inheritance setting will - not apply to parent directories if they must be created + inheritance (bool): + If True the object will inherit permissions from the parent, if + ``False``, inheritance will be disabled. Inheritance setting will + not apply to parent directories if they must be created. + + reset (bool): + If ``True`` the existing DACL will be cleared and replaced with the + settings defined in this function. If ``False``, new entries will be + appended to the existing DACL. Default is ``False``. + + .. versionadded:: Oxygen Returns: bool: True if successful @@ -1289,10 +1303,16 @@ def mkdir(path, # Set owner if owner: - salt.utils.win_dacl.set_owner(path, owner) + salt.utils.win_dacl.set_owner(obj_name=path, principal=owner) # Set permissions - set_perms(path, grant_perms, deny_perms, inheritance) + set_perms( + path=path, + grant_perms=grant_perms, + deny_perms=deny_perms, + inheritance=inheritance, + reset=reset) + except WindowsError as exc: raise CommandExecutionError(exc) @@ -1303,49 +1323,63 @@ def makedirs_(path, owner=None, grant_perms=None, deny_perms=None, - inheritance=True): + inheritance=True, + reset=False): ''' Ensure that the parent directory containing this path is available. Args: - path (str): The full path to the directory. + path (str): + The full path to the directory. - owner (str): The owner of the directory. If not passed, it will be the - account that created the directly, likely SYSTEM + .. note:: - grant_perms (dict): A dictionary containing the user/group and the basic - permissions to grant, ie: ``{'user': {'perms': 'basic_permission'}}``. - You can also set the ``applies_to`` setting here. The default is - ``this_folder_subfolders_files``. Specify another ``applies_to`` setting - like this: + The path must end with a trailing slash otherwise the + directory(s) will be created up to the parent directory. For + example if path is ``C:\\temp\\test``, then it would be treated + as ``C:\\temp\\`` but if the path ends with a trailing slash + like ``C:\\temp\\test\\``, then it would be treated as + ``C:\\temp\\test\\``. - .. code-block:: yaml + owner (str): + The owner of the directory. If not passed, it will be the account + that created the directly, likely SYSTEM - {'user': {'perms': 'full_control', 'applies_to': 'this_folder'}} + grant_perms (dict): + A dictionary containing the user/group and the basic permissions to + grant, ie: ``{'user': {'perms': 'basic_permission'}}``. You can also + set the ``applies_to`` setting here. The default is + ``this_folder_subfolders_files``. Specify another ``applies_to`` + setting like this: - To set advanced permissions use a list for the ``perms`` parameter, ie: + .. code-block:: yaml - .. code-block:: yaml + {'user': {'perms': 'full_control', 'applies_to': 'this_folder'}} - {'user': {'perms': ['read_attributes', 'read_ea'], 'applies_to': 'this_folder'}} + To set advanced permissions use a list for the ``perms`` parameter, ie: - deny_perms (dict): A dictionary containing the user/group and - permissions to deny along with the ``applies_to`` setting. Use the same - format used for the ``grant_perms`` parameter. Remember, deny - permissions supersede grant permissions. + .. code-block:: yaml - inheritance (bool): If True the object will inherit permissions from the - parent, if False, inheritance will be disabled. Inheritance setting will - not apply to parent directories if they must be created + {'user': {'perms': ['read_attributes', 'read_ea'], 'applies_to': 'this_folder'}} - .. note:: + deny_perms (dict): + A dictionary containing the user/group and permissions to deny along + with the ``applies_to`` setting. Use the same format used for the + ``grant_perms`` parameter. Remember, deny permissions supersede + grant permissions. - The path must end with a trailing slash otherwise the directory(s) will - be created up to the parent directory. For example if path is - ``C:\\temp\\test``, then it would be treated as ``C:\\temp\\`` but if - the path ends with a trailing slash like ``C:\\temp\\test\\``, then it - would be treated as ``C:\\temp\\test\\``. + inheritance (bool): + If True the object will inherit permissions from the parent, if + False, inheritance will be disabled. Inheritance setting will not + apply to parent directories if they must be created. + + reset (bool): + If ``True`` the existing DACL will be cleared and replaced with the + settings defined in this function. If ``False``, new entries will be + appended to the existing DACL. Default is ``False``. + + .. versionadded:: Oxygen Returns: bool: True if successful @@ -1405,7 +1439,13 @@ def makedirs_(path, for directory_to_create in directories_to_create: # all directories have the user, group and mode set!! log.debug('Creating directory: %s', directory_to_create) - mkdir(directory_to_create, owner, grant_perms, deny_perms, inheritance) + mkdir( + path=directory_to_create, + owner=owner, + grant_perms=grant_perms, + deny_perms=deny_perms, + inheritance=inheritance, + reset=reset) return True @@ -1414,41 +1454,54 @@ def makedirs_perms(path, owner=None, grant_perms=None, deny_perms=None, - inheritance=True): + inheritance=True, + reset=True): ''' Set owner and permissions for each directory created. Args: - path (str): The full path to the directory. + path (str): + The full path to the directory. - owner (str): The owner of the directory. If not passed, it will be the - account that created the directory, likely SYSTEM + owner (str): + The owner of the directory. If not passed, it will be the account + that created the directory, likely SYSTEM - grant_perms (dict): A dictionary containing the user/group and the basic - permissions to grant, ie: ``{'user': {'perms': 'basic_permission'}}``. - You can also set the ``applies_to`` setting here. The default is - ``this_folder_subfolders_files``. Specify another ``applies_to`` setting - like this: + grant_perms (dict): + A dictionary containing the user/group and the basic permissions to + grant, ie: ``{'user': {'perms': 'basic_permission'}}``. You can also + set the ``applies_to`` setting here. The default is + ``this_folder_subfolders_files``. Specify another ``applies_to`` + setting like this: - .. code-block:: yaml + .. code-block:: yaml - {'user': {'perms': 'full_control', 'applies_to': 'this_folder'}} + {'user': {'perms': 'full_control', 'applies_to': 'this_folder'}} - To set advanced permissions use a list for the ``perms`` parameter, ie: + To set advanced permissions use a list for the ``perms`` parameter, ie: - .. code-block:: yaml + .. code-block:: yaml - {'user': {'perms': ['read_attributes', 'read_ea'], 'applies_to': 'this_folder'}} + {'user': {'perms': ['read_attributes', 'read_ea'], 'applies_to': 'this_folder'}} - deny_perms (dict): A dictionary containing the user/group and - permissions to deny along with the ``applies_to`` setting. Use the same - format used for the ``grant_perms`` parameter. Remember, deny - permissions supersede grant permissions. + deny_perms (dict): + A dictionary containing the user/group and permissions to deny along + with the ``applies_to`` setting. Use the same format used for the + ``grant_perms`` parameter. Remember, deny permissions supersede + grant permissions. - inheritance (bool): If True the object will inherit permissions from the - parent, if False, inheritance will be disabled. Inheritance setting will - not apply to parent directories if they must be created + inheritance (bool): + If ``True`` the object will inherit permissions from the parent, if + ``False``, inheritance will be disabled. Inheritance setting will + not apply to parent directories if they must be created + + reset (bool): + If ``True`` the existing DACL will be cleared and replaced with the + settings defined in this function. If ``False``, new entries will be + appended to the existing DACL. Default is ``False``. + + .. versionadded:: Oxygen Returns: bool: True if successful, otherwise raise an error @@ -1482,8 +1535,15 @@ def makedirs_perms(path, try: # Create the directory here, set inherited True because this is a # parent directory, the inheritance setting will only apply to the - # child directory - makedirs_perms(head, owner, grant_perms, deny_perms, True) + # target directory. Reset will be False as we only want to reset + # the permissions on the target directory + makedirs_perms( + path=head, + owner=owner, + grant_perms=grant_perms, + deny_perms=deny_perms, + inheritance=True, + reset=False) except OSError as exc: # be happy if someone already created the path if exc.errno != errno.EEXIST: @@ -1492,7 +1552,13 @@ def makedirs_perms(path, return {} # Make the directory - mkdir(path, owner, grant_perms, deny_perms, inheritance) + mkdir( + path=path, + owner=owner, + grant_perms=grant_perms, + deny_perms=deny_perms, + inheritance=inheritance, + reset=reset) return True @@ -1502,66 +1568,64 @@ def check_perms(path, owner=None, grant_perms=None, deny_perms=None, - inheritance=True): + inheritance=True, + reset=False): ''' - Set owner and permissions for each directory created. + Check owner and permissions for the passed directory. This function checks + the permissions and sets them, returning the changes made. Args: - path (str): The full path to the directory. + path (str): + The full path to the directory. - ret (dict): A dictionary to append changes to and return. If not passed, - will create a new dictionary to return. + ret (dict): + A dictionary to append changes to and return. If not passed, will + create a new dictionary to return. - owner (str): The owner of the directory. If not passed, it will be the - account that created the directory, likely SYSTEM + owner (str): + The owner to set for the directory. - grant_perms (dict): A dictionary containing the user/group and the basic - permissions to grant, ie: ``{'user': {'perms': 'basic_permission'}}``. - You can also set the ``applies_to`` setting here. The default is - ``this_folder_subfolders_files``. Specify another ``applies_to`` setting - like this: + grant_perms (dict): + A dictionary containing the user/group and the basic permissions to + check/grant, ie: ``{'user': {'perms': 'basic_permission'}}``. + Default is ``None``. - .. code-block:: yaml + deny_perms (dict): + A dictionary containing the user/group and permissions to + check/deny. Default is ``None``. - {'user': {'perms': 'full_control', 'applies_to': 'this_folder'}} + inheritance (bool): + ``True will check if inheritance is enabled and enable it. ``False`` + will check if inheritance is disabled and disable it. Defaultl is + ``True``. - To set advanced permissions use a list for the ``perms`` parameter, ie: - - .. code-block:: yaml - - {'user': {'perms': ['read_attributes', 'read_ea'], 'applies_to': 'this_folder'}} - - deny_perms (dict): A dictionary containing the user/group and - permissions to deny along with the ``applies_to`` setting. Use the same - format used for the ``grant_perms`` parameter. Remember, deny - permissions supersede grant permissions. - - inheritance (bool): If True the object will inherit permissions from the - parent, if False, inheritance will be disabled. Inheritance setting will - not apply to parent directories if they must be created + reset (bool): + ``True`` wil show what permisisons will be removed by resetting the + DACL. ``False`` will do nothing. Default is ``False``. Returns: - bool: True if successful, otherwise raise an error + dict: A dictionary of changes that have been made CLI Example: .. code-block:: bash - # To grant the 'Users' group 'read & execute' permissions. - salt '*' file.check_perms C:\\Temp\\ Administrators "{'Users': {'perms': 'read_execute'}}" + # To see changes to ``C:\\Temp`` if the 'Users' group is given 'read & execute' permissions. + salt '*' file.check_perms C:\\Temp\\ {} Administrators "{'Users': {'perms': 'read_execute'}}" # Locally using salt call - salt-call file.check_perms C:\\Temp\\ Administrators "{'Users': {'perms': 'read_execute', 'applies_to': 'this_folder_only'}}" + salt-call file.check_perms C:\\Temp\\ {} Administrators "{'Users': {'perms': 'read_execute', 'applies_to': 'this_folder_only'}}" # Specify advanced attributes with a list - salt '*' file.check_perms C:\\Temp\\ Administrators "{'jsnuffy': {'perms': ['read_attributes', 'read_ea'], 'applies_to': 'files_only'}}" + salt '*' file.check_perms C:\\Temp\\ {} Administrators "{'jsnuffy': {'perms': ['read_attributes', 'read_ea'], 'applies_to': 'files_only'}}" ''' path = os.path.expanduser(path) if not ret: ret = {'name': path, 'changes': {}, + 'pchanges': {}, 'comment': [], 'result': True} orig_comment = '' @@ -1571,14 +1635,16 @@ def check_perms(path, # Check owner if owner: - owner = salt.utils.win_dacl.get_name(owner) - current_owner = salt.utils.win_dacl.get_owner(path) + owner = salt.utils.win_dacl.get_name(principal=owner) + current_owner = salt.utils.win_dacl.get_owner(obj_name=path) if owner != current_owner: if __opts__['test'] is True: ret['pchanges']['owner'] = owner else: try: - salt.utils.win_dacl.set_owner(path, owner) + salt.utils.win_dacl.set_owner( + obj_name=path, + principal=owner) ret['changes']['owner'] = owner except CommandExecutionError: ret['result'] = False @@ -1586,7 +1652,7 @@ def check_perms(path, 'Failed to change owner to "{0}"'.format(owner)) # Check permissions - cur_perms = salt.utils.win_dacl.get_permissions(path) + cur_perms = salt.utils.win_dacl.get_permissions(obj_name=path) # Verify Deny Permissions changes = {} @@ -1594,7 +1660,7 @@ def check_perms(path, for user in deny_perms: # Check that user exists: try: - user_name = salt.utils.win_dacl.get_name(user) + user_name = salt.utils.win_dacl.get_name(principal=user) except CommandExecutionError: ret['comment'].append( 'Deny Perms: User "{0}" missing from Target System'.format(user)) @@ -1619,7 +1685,11 @@ def check_perms(path, # Check Perms if isinstance(deny_perms[user]['perms'], six.string_types): if not salt.utils.win_dacl.has_permission( - path, user, deny_perms[user]['perms'], 'deny'): + obj_name=path, + principal=user, + permission=deny_perms[user]['perms'], + access_mode='deny', + exact=False): changes[user] = {'perms': deny_perms[user]['perms']} else: for perm in deny_perms[user]['perms']: @@ -1640,9 +1710,10 @@ def check_perms(path, changes[user]['applies_to'] = applies_to if changes: + ret['pchanges']['deny_perms'] = {} ret['changes']['deny_perms'] = {} for user in changes: - user_name = salt.utils.win_dacl.get_name(user) + user_name = salt.utils.win_dacl.get_name(principal=user) if __opts__['test'] is True: ret['pchanges']['deny_perms'][user] = changes[user] @@ -1689,7 +1760,11 @@ def check_perms(path, try: salt.utils.win_dacl.set_permissions( - path, user, perms, 'deny', applies_to) + obj_name=path, + principal=user, + permissions=perms, + access_mode='deny', + applies_to=applies_to) ret['changes']['deny_perms'][user] = changes[user] except CommandExecutionError: ret['result'] = False @@ -1703,7 +1778,7 @@ def check_perms(path, for user in grant_perms: # Check that user exists: try: - user_name = salt.utils.win_dacl.get_name(user) + user_name = salt.utils.win_dacl.get_name(principal=user) except CommandExecutionError: ret['comment'].append( 'Grant Perms: User "{0}" missing from Target System'.format(user)) @@ -1729,12 +1804,19 @@ def check_perms(path, # Check Perms if isinstance(grant_perms[user]['perms'], six.string_types): if not salt.utils.win_dacl.has_permission( - path, user, grant_perms[user]['perms']): + obj_name=path, + principal=user, + permission=grant_perms[user]['perms'], + access_mode='grant'): changes[user] = {'perms': grant_perms[user]['perms']} else: for perm in grant_perms[user]['perms']: if not salt.utils.win_dacl.has_permission( - path, user, perm, exact=False): + obj_name=path, + principal=user, + permission=perm, + access_mode='grant', + exact=False): if user not in changes: changes[user] = {'perms': []} changes[user]['perms'].append(grant_perms[user]['perms']) @@ -1750,11 +1832,12 @@ def check_perms(path, changes[user]['applies_to'] = applies_to if changes: + ret['pchanges']['grant_perms'] = {} ret['changes']['grant_perms'] = {} for user in changes: - user_name = salt.utils.win_dacl.get_name(user) + user_name = salt.utils.win_dacl.get_name(principal=user) if __opts__['test'] is True: - ret['changes']['grant_perms'][user] = changes[user] + ret['pchanges']['grant_perms'][user] = changes[user] else: applies_to = None if 'applies_to' not in changes[user]: @@ -1796,7 +1879,11 @@ def check_perms(path, try: salt.utils.win_dacl.set_permissions( - path, user, perms, 'grant', applies_to) + obj_name=path, + principal=user, + permissions=perms, + access_mode='grant', + applies_to=applies_to) ret['changes']['grant_perms'][user] = changes[user] except CommandExecutionError: ret['result'] = False @@ -1806,12 +1893,14 @@ def check_perms(path, # Check inheritance if inheritance is not None: - if not inheritance == salt.utils.win_dacl.get_inheritance(path): + if not inheritance == salt.utils.win_dacl.get_inheritance(obj_name=path): if __opts__['test'] is True: - ret['changes']['inheritance'] = inheritance + ret['pchanges']['inheritance'] = inheritance else: try: - salt.utils.win_dacl.set_inheritance(path, inheritance) + salt.utils.win_dacl.set_inheritance( + obj_name=path, + enabled=inheritance) ret['changes']['inheritance'] = inheritance except CommandExecutionError: ret['result'] = False @@ -1819,6 +1908,45 @@ def check_perms(path, 'Failed to set inheritance for "{0}" to ' '{1}'.format(path, inheritance)) + # Check reset + # If reset=True, which users will be removed as a result + if reset: + for user_name in cur_perms: + if user_name not in grant_perms: + if 'grant' in cur_perms[user_name] and not \ + cur_perms[user_name]['grant']['inherited']: + if __opts__['test'] is True: + if 'remove_perms' not in ret['pchanges']: + ret['pchanges']['remove_perms'] = {} + ret['pchanges']['remove_perms'].update( + {user_name: cur_perms[user_name]}) + else: + if 'remove_perms' not in ret['changes']: + ret['changes']['remove_perms'] = {} + salt.utils.win_dacl.rm_permissions( + obj_name=path, + principal=user_name, + ace_type='grant') + ret['changes']['remove_perms'].update( + {user_name: cur_perms[user_name]}) + if user_name not in deny_perms: + if 'deny' in cur_perms[user_name] and not \ + cur_perms[user_name]['deny']['inherited']: + if __opts__['test'] is True: + if 'remove_perms' not in ret['pchanges']: + ret['pchanges']['remove_perms'] = {} + ret['pchanges']['remove_perms'].update( + {user_name: cur_perms[user_name]}) + else: + if 'remove_perms' not in ret['changes']: + ret['changes']['remove_perms'] = {} + salt.utils.win_dacl.rm_permissions( + obj_name=path, + principal=user_name, + ace_type='deny') + ret['changes']['remove_perms'].update( + {user_name: cur_perms[user_name]}) + # Re-add the Original Comment if defined if isinstance(orig_comment, six.string_types): if orig_comment: @@ -1830,25 +1958,30 @@ def check_perms(path, ret['comment'] = '\n'.join(ret['comment']) # Set result for test = True - if __opts__['test'] is True and ret['changes']: + if __opts__['test'] and (ret['changes'] or ret['pchanges']): ret['result'] = None return ret -def set_perms(path, grant_perms=None, deny_perms=None, inheritance=True): +def set_perms(path, + grant_perms=None, + deny_perms=None, + inheritance=True, + reset=False): ''' Set permissions for the given path Args: - path (str): The full path to the directory. + path (str): + The full path to the directory. grant_perms (dict): A dictionary containing the user/group and the basic permissions to grant, ie: ``{'user': {'perms': 'basic_permission'}}``. You can also - set the ``applies_to`` setting here. The default is - ``this_folder_subfolders_files``. Specify another ``applies_to`` + set the ``applies_to`` setting here. The default for ``applise_to`` + is ``this_folder_subfolders_files``. Specify another ``applies_to`` setting like this: .. code-block:: yaml @@ -1863,7 +1996,10 @@ def set_perms(path, grant_perms=None, deny_perms=None, inheritance=True): {'user': {'perms': ['read_attributes', 'read_ea'], 'applies_to': 'this_folder'}} To see a list of available attributes and applies to settings see - the documentation for salt.utils.win_dacl + the documentation for salt.utils.win_dacl. + + A value of ``None`` will make no changes to the ``grant`` portion of + the DACL. Default is ``None``. deny_perms (dict): A dictionary containing the user/group and permissions to deny along @@ -1871,13 +2007,27 @@ def set_perms(path, grant_perms=None, deny_perms=None, inheritance=True): ``grant_perms`` parameter. Remember, deny permissions supersede grant permissions. + A value of ``None`` will make no changes to the ``deny`` portion of + the DACL. Default is ``None``. + inheritance (bool): - If True the object will inherit permissions from the parent, if - False, inheritance will be disabled. Inheritance setting will not - apply to parent directories if they must be created + If ``True`` the object will inherit permissions from the parent, if + ``False``, inheritance will be disabled. Inheritance setting will + not apply to parent directories if they must be created. Default is + ``False``. + + reset (bool): + If ``True`` the existing DCL will be cleared and replaced with the + settings defined in this function. If ``False``, new entries will be + appended to the existing DACL. Default is ``False``. + + .. versionadded: Oxygen Returns: - bool: True if successful, otherwise raise an error + bool: True if successful + + Raises: + CommandExecutionError: If unsuccessful CLI Example: @@ -1894,11 +2044,19 @@ def set_perms(path, grant_perms=None, deny_perms=None, inheritance=True): ''' ret = {} - # Get the DACL for the directory - dacl = salt.utils.win_dacl.dacl(path) + if reset: + # Get an empty DACL + dacl = salt.utils.win_dacl.dacl() - # Get current file/folder permissions - cur_perms = salt.utils.win_dacl.get_permissions(path) + # Get an empty perms dict + cur_perms = {} + + else: + # Get the DACL for the directory + dacl = salt.utils.win_dacl.dacl(path) + + # Get current file/folder permissions + cur_perms = salt.utils.win_dacl.get_permissions(path) # Set 'deny' perms if any if deny_perms is not None: diff --git a/salt/states/file.py b/salt/states/file.py index ec4f7b16148..28aff082e2e 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -761,7 +761,8 @@ def _check_directory_win(name, win_owner, win_perms=None, win_deny_perms=None, - win_inheritance=None): + win_inheritance=None, + win_perms_reset=None): ''' Check what changes need to be made on a directory ''' @@ -879,6 +880,20 @@ def _check_directory_win(name, if not win_inheritance == salt.utils.win_dacl.get_inheritance(name): changes['inheritance'] = win_inheritance + # Check reset + if win_perms_reset: + for user_name in perms: + if user_name not in win_perms: + if 'grant' in perms[user_name] and not perms[user_name]['grant']['inherited']: + if 'remove_perms' not in changes: + changes['remove_perms'] = {} + changes['remove_perms'].update({user_name: perms[user_name]}) + if user_name not in win_deny_perms: + if 'deny' in perms[user_name] and not perms[user_name]['deny']['inherited']: + if 'remove_perms' not in changes: + changes['remove_perms'] = {} + changes['remove_perms'].update({user_name: perms[user_name]}) + if changes: return None, 'The directory "{0}" will be changed'.format(name), changes @@ -1566,6 +1581,7 @@ def managed(name, win_perms=None, win_deny_perms=None, win_inheritance=True, + win_perms_reset=False, **kwargs): r''' Manage a given file, this function allows for a file to be downloaded from @@ -2072,6 +2088,13 @@ def managed(name, .. versionadded:: 2017.7.0 + win_perms_reset : False + If ``True`` the existing DACL will be cleared and replaced with the + settings defined in this function. If ``False``, new entries will be + appended to the existing DACL. Default is ``False``. + + .. versionadded:: Oxygen + Here's an example using the above ``win_*`` parameters: .. code-block:: yaml @@ -2314,8 +2337,13 @@ def managed(name, # Check and set the permissions if necessary if salt.utils.platform.is_windows(): ret = __salt__['file.check_perms']( - name, ret, win_owner, win_perms, win_deny_perms, None, - win_inheritance) + path=name, + ret=ret, + owner=win_owner, + grant_perms=win_perms, + deny_perms=win_deny_perms, + inheritance=win_inheritance, + reset=win_perms_reset) else: ret, _ = __salt__['file.check_perms']( name, ret, user, group, mode, attrs, follow_symlinks) @@ -2356,8 +2384,13 @@ def managed(name, if salt.utils.platform.is_windows(): ret = __salt__['file.check_perms']( - name, ret, win_owner, win_perms, win_deny_perms, None, - win_inheritance) + path=name, + ret=ret, + owner=win_owner, + grant_perms=win_perms, + deny_perms=win_deny_perms, + inheritance=win_inheritance, + reset=win_perms_reset) if isinstance(ret['pchanges'], tuple): ret['result'], ret['comment'] = ret['pchanges'] @@ -2448,6 +2481,7 @@ def managed(name, win_perms=win_perms, win_deny_perms=win_deny_perms, win_inheritance=win_inheritance, + win_perms_reset=win_perms_reset, encoding=encoding, encoding_errors=encoding_errors, **kwargs) @@ -2517,6 +2551,7 @@ def managed(name, win_perms=win_perms, win_deny_perms=win_deny_perms, win_inheritance=win_inheritance, + win_perms_reset=win_perms_reset, encoding=encoding, encoding_errors=encoding_errors, **kwargs) @@ -2590,6 +2625,7 @@ def directory(name, win_perms=None, win_deny_perms=None, win_inheritance=True, + win_perms_reset=False, **kwargs): r''' Ensure that a named directory is present and has the right perms @@ -2751,6 +2787,13 @@ def directory(name, .. versionadded:: 2017.7.0 + win_perms_reset : False + If ``True`` the existing DACL will be cleared and replaced with the + settings defined in this function. If ``False``, new entries will be + appended to the existing DACL. Default is ``False``. + + .. versionadded:: Oxygen + Here's an example using the above ``win_*`` parameters: .. code-block:: yaml @@ -2855,13 +2898,23 @@ def directory(name, elif force: # Remove whatever is in the way if os.path.isfile(name): - os.remove(name) - ret['changes']['forced'] = 'File was forcibly replaced' + if __opts__['test']: + ret['pchanges']['forced'] = 'File was forcibly replaced' + else: + os.remove(name) + ret['changes']['forced'] = 'File was forcibly replaced' elif __salt__['file.is_link'](name): - __salt__['file.remove'](name) - ret['changes']['forced'] = 'Symlink was forcibly replaced' + if __opts__['test']: + ret['pchanges']['forced'] = 'Symlink was forcibly replaced' + else: + __salt__['file.remove'](name) + ret['changes']['forced'] = 'Symlink was forcibly replaced' else: - __salt__['file.remove'](name) + if __opts__['test']: + ret['pchanges']['forced'] = 'Directory was forcibly replaced' + else: + __salt__['file.remove'](name) + ret['changes']['forced'] = 'Directory was forcibly replaced' else: if os.path.isfile(name): return _error( @@ -2874,17 +2927,26 @@ def directory(name, # Check directory? if salt.utils.platform.is_windows(): - presult, pcomment, ret['pchanges'] = _check_directory_win( - name, win_owner, win_perms, win_deny_perms, win_inheritance) + presult, pcomment, pchanges = _check_directory_win( + name=name, + win_owner=win_owner, + win_perms=win_perms, + win_deny_perms=win_deny_perms, + win_inheritance=win_inheritance, + win_perms_reset=win_perms_reset) else: - presult, pcomment, ret['pchanges'] = _check_directory( + presult, pcomment, pchanges = _check_directory( name, user, group, recurse or [], dir_mode, clean, require, exclude_pat, max_depth, follow_symlinks) - if __opts__['test']: + if pchanges: + ret['pchanges'].update(pchanges) + + # Don't run through the reset of the function if there are no changes to be + # made + if not ret['pchanges'] or __opts__['test']: ret['result'] = presult ret['comment'] = pcomment - ret['changes'] = ret['pchanges'] return ret if not os.path.isdir(name): @@ -2900,8 +2962,13 @@ def directory(name, if not os.path.isdir(drive): return _error( ret, 'Drive {0} is not mapped'.format(drive)) - __salt__['file.makedirs'](name, win_owner, win_perms, - win_deny_perms, win_inheritance) + __salt__['file.makedirs']( + path=name, + owner=win_owner, + grant_perms=win_perms, + deny_perms=win_deny_perms, + inheritance=win_inheritance, + reset=win_perms_reset) else: __salt__['file.makedirs'](name, user=user, group=group, mode=dir_mode) @@ -2910,8 +2977,13 @@ def directory(name, ret, 'No directory to create {0} in'.format(name)) if salt.utils.platform.is_windows(): - __salt__['file.mkdir'](name, win_owner, win_perms, win_deny_perms, - win_inheritance) + __salt__['file.mkdir']( + path=name, + owner=win_owner, + grant_perms=win_perms, + deny_perms=win_deny_perms, + inheritance=win_inheritance, + reset=win_perms_reset) else: __salt__['file.mkdir'](name, user=user, group=group, mode=dir_mode) @@ -2925,7 +2997,13 @@ def directory(name, if not children_only: if salt.utils.platform.is_windows(): ret = __salt__['file.check_perms']( - name, ret, win_owner, win_perms, win_deny_perms, None, win_inheritance) + path=name, + ret=ret, + owner=win_owner, + grant_perms=win_perms, + deny_perms=win_deny_perms, + inheritance=win_inheritance, + reset=win_perms_reset) else: ret, perms = __salt__['file.check_perms']( name, ret, user, group, dir_mode, None, follow_symlinks) @@ -2996,8 +3074,13 @@ def directory(name, try: if salt.utils.platform.is_windows(): ret = __salt__['file.check_perms']( - full, ret, win_owner, win_perms, win_deny_perms, None, - win_inheritance) + path=full, + ret=ret, + owner=win_owner, + grant_perms=win_perms, + deny_perms=win_deny_perms, + inheritance=win_inheritance, + reset=win_perms_reset) else: ret, _ = __salt__['file.check_perms']( full, ret, user, group, file_mode, None, follow_symlinks) @@ -3011,8 +3094,13 @@ def directory(name, try: if salt.utils.platform.is_windows(): ret = __salt__['file.check_perms']( - full, ret, win_owner, win_perms, win_deny_perms, None, - win_inheritance) + path=full, + ret=ret, + owner=win_owner, + grant_perms=win_perms, + deny_perms=win_deny_perms, + inheritance=win_inheritance, + reset=win_perms_reset) else: ret, _ = __salt__['file.check_perms']( full, ret, user, group, dir_mode, None, follow_symlinks) @@ -3034,7 +3122,8 @@ def directory(name, if children_only: ret['comment'] = u'Directory {0}/* updated'.format(name) else: - ret['comment'] = u'Directory {0} updated'.format(name) + if ret['changes']: + ret['comment'] = u'Directory {0} updated'.format(name) if __opts__['test']: ret['comment'] = 'Directory {0} not updated'.format(name) diff --git a/tests/unit/states/test_file.py b/tests/unit/states/test_file.py index 1e53c0c3f5d..86fb0497652 100644 --- a/tests/unit/states/test_file.py +++ b/tests/unit/states/test_file.py @@ -815,7 +815,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'comment': comt, 'result': None, 'pchanges': p_chg, - 'changes': {'/etc/grub.conf': {'directory': 'new'}} + 'changes': {} }) self.assertDictEqual(filestate.directory(name, user=user, @@ -841,6 +841,11 @@ class TestFileState(TestCase, LoaderModuleMockMixin): ret) recurse = ['ignore_files', 'ignore_dirs'] + ret.update({'comment': 'Must not specify "recurse" ' + 'options "ignore_files" and ' + '"ignore_dirs" at the same ' + 'time.', + 'pchanges': {}}) with patch.object(os.path, 'isdir', mock_t): self.assertDictEqual(filestate.directory (name, user=user, From 281992195a5aedd0b0d198ad5fbfb56dc52cf126 Mon Sep 17 00:00:00 2001 From: Adam Mendlik Date: Sat, 2 Dec 2017 17:30:41 -0700 Subject: [PATCH 138/159] Correct CLI examples in SDB runner documentation --- salt/runners/sdb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/runners/sdb.py b/salt/runners/sdb.py index e70f621c00b..d8a8f37adcf 100644 --- a/salt/runners/sdb.py +++ b/salt/runners/sdb.py @@ -22,7 +22,7 @@ def get(uri): .. code-block:: bash - salt '*' sdb.get sdb://mymemcached/foo + salt-run sdb.get sdb://mymemcached/foo ''' return salt.utils.sdb.sdb_get(uri, __opts__, __utils__) @@ -37,7 +37,7 @@ def set_(uri, value): .. code-block:: bash - salt '*' sdb.set sdb://mymemcached/foo bar + salt-run sdb.set sdb://mymemcached/foo bar ''' return salt.utils.sdb.sdb_set(uri, value, __opts__, __utils__) @@ -52,7 +52,7 @@ def delete(uri): .. code-block:: bash - salt '*' sdb.delete sdb://mymemcached/foo + salt-run sdb.delete sdb://mymemcached/foo ''' return salt.utils.sdb.sdb_delete(uri, __opts__, __utils__) From 1eb4fc884c7e6539992295cc958baa982c9509f6 Mon Sep 17 00:00:00 2001 From: Jorge Schrauwen Date: Sun, 3 Dec 2017 10:25:29 +0100 Subject: [PATCH 139/159] Fix swap grains for Solaris-like operating systems The output of ```swap -s``` is the same on atleast the following: - Solaris 11.3 - Solaris 10 - OmniOS - OpenIndiana - SmartOS ``` total: 3251464k bytes allocated + 827664k reserved = 4079128k used, 181507256k available ``` The first value is the current allocate mount of swap without reservations, the 2nd value is the reservation, the 3rd value is the current used (allocate + reservation) the last value is the amount of currently available swap. We want available + used to get the total. --- salt/grains/core.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/salt/grains/core.py b/salt/grains/core.py index da541edf56c..e3e47c62a57 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -474,8 +474,14 @@ def _sunos_memdata(): grains['mem_total'] = int(comps[2].strip()) swap_cmd = salt.utils.path.which('swap') - swap_total = __salt__['cmd.run']('{0} -s'.format(swap_cmd)).split()[1] - grains['swap_total'] = int(swap_total) // 1024 + swap_data = __salt__['cmd.run']('{0} -s'.format(swap_cmd)).split() + try: + swap_avail = int(swap_data[-2][:-1]) + swap_used = int(swap_data[-4][:-1]) + swap_total = (swap_avail + swap_used) // 1024 + except ValueError: + swap_total = None + grains['swap_total'] = swap_total return grains From b811cc7343e02eb0f8ed13bd552b132870320777 Mon Sep 17 00:00:00 2001 From: Volodymyr Samodid Date: Sun, 3 Dec 2017 22:38:26 +0200 Subject: [PATCH 140/159] fix copy-paste errors in doc strings for saltutil module and runner --- salt/modules/saltutil.py | 4 ++-- salt/runners/saltutil.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/salt/modules/saltutil.py b/salt/modules/saltutil.py index 12f8be9fa46..27d7a44141c 100644 --- a/salt/modules/saltutil.py +++ b/salt/modules/saltutil.py @@ -474,7 +474,7 @@ def sync_returners(saltenv=None, refresh=True, extmod_whitelist=None, extmod_bla ''' .. versionadded:: 0.10.0 - Sync beacons from ``salt://_returners`` to the minion + Sync returners from ``salt://_returners`` to the minion saltenv The fileserver environment from which to sync. To sync from more than @@ -628,7 +628,7 @@ def sync_clouds(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blackl ''' .. versionadded:: 2017.7.0 - Sync utility modules from ``salt://_cloud`` to the minion + Sync cloud modules from ``salt://_cloud`` to the minion saltenv : base The fileserver environment from which to sync. To sync from more than diff --git a/salt/runners/saltutil.py b/salt/runners/saltutil.py index b691f827e1d..88173eb291e 100644 --- a/salt/runners/saltutil.py +++ b/salt/runners/saltutil.py @@ -381,7 +381,7 @@ def sync_sdb(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): ''' .. versionadded:: 2017.7.0 - Sync utils modules from ``salt://_sdb`` to the master + Sync sdb modules from ``salt://_sdb`` to the master saltenv : base The fileserver environment from which to sync. To sync from more than @@ -427,7 +427,7 @@ def sync_cache(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): ''' .. versionadded:: 2017.7.0 - Sync utils modules from ``salt://_cache`` to the master + Sync cache modules from ``salt://_cache`` to the master saltenv : base The fileserver environment from which to sync. To sync from more than @@ -453,7 +453,7 @@ def sync_fileserver(saltenv='base', extmod_whitelist=None, extmod_blacklist=None ''' .. versionadded:: Oxygen - Sync utils modules from ``salt://_fileserver`` to the master + Sync fileserver modules from ``salt://_fileserver`` to the master saltenv : base The fileserver environment from which to sync. To sync from more than @@ -479,7 +479,7 @@ def sync_clouds(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): ''' .. versionadded:: 2017.7.0 - Sync utils modules from ``salt://_clouds`` to the master + Sync cloud modules from ``salt://_clouds`` to the master saltenv : base The fileserver environment from which to sync. To sync from more than @@ -505,7 +505,7 @@ def sync_roster(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): ''' .. versionadded:: 2017.7.0 - Sync utils modules from ``salt://_roster`` to the master + Sync roster modules from ``salt://_roster`` to the master saltenv : base The fileserver environment from which to sync. To sync from more than From b8db7ab7bf4edee2613d20d0c09fda21cddfc3c1 Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Tue, 14 Nov 2017 09:04:06 +0100 Subject: [PATCH 141/159] support more options for zypper search module --- salt/modules/zypper.py | 88 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 6 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index 71b280c0400..6c717b9818c 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -1750,7 +1750,7 @@ def list_installed_patterns(): return _get_patterns(installed_only=True) -def search(criteria, refresh=False): +def search(criteria, refresh=False, **kwargs): ''' List known packags, available to the system. @@ -1759,6 +1759,47 @@ def search(criteria, refresh=False): If set to False (default) it depends on zypper if a refresh is executed. + match (str) + One of `exact`, `words`, `substrings`. Search for an `exact` match + or for the whole `words` only. Default to `substrings` to patch + partial words. + + provides (bool) + Search for packages which provide the search strings. + + recommends (bool) + Search for packages which recommend the search strings. + + requires (bool) + Search for packages which require the search strings. + + suggests (bool) + Search for packages which suggest the search strings. + + conflicts (bool) + Search packages conflicting with search strings. + + obsoletes (bool) + Search for packages which obsolete the search strings. + + file_list (bool) + Search for a match in the file list of packages. + + search_descriptions (bool) + Search also in package summaries and descriptions. + + case_sensitive (bool) + Perform case-sensitive search. + + installed_only (bool) + Show only installed packages. + + not_installed_only (bool) + Show only packages which are not installed. + + details (bool) + Show version and repository + CLI Examples: .. code-block:: bash @@ -1768,17 +1809,52 @@ def search(criteria, refresh=False): if refresh: refresh_db() - solvables = __zypper__.nolock.xml.call('se', criteria).getElementsByTagName('solvable') + cmd = ['se'] + if kwargs.get('match') == 'exact': + cmd.append('--match-exact') + elif kwargs.get('match') == 'words': + cmd.append('--match-words') + elif kwargs.get('match') == 'substrings': + cmd.append('--match-substrings') + + if kwargs.get('provides'): + cmd.append('--provides') + if kwargs.get('recommends'): + cmd.append('--recommends') + if kwargs.get('requires'): + cmd.append('--requires') + if kwargs.get('suggests'): + cmd.append('--suggests') + if kwargs.get('conflicts'): + cmd.append('--conflicts') + if kwargs.get('obsoletes'): + cmd.append('--obsoletes') + + if kwargs.get('file_list'): + cmd.append('--file-list') + if kwargs.get('search_descriptions'): + cmd.append('--search-descriptions') + if kwargs.get('case_sensitive'): + cmd.append('--case-sensitive') + if kwargs.get('installed_only'): + cmd.append('--installed-only') + if kwargs.get('not_installed_only'): + cmd.append('--not-installed-only') + if kwargs.get('details'): + cmd.append('--details') + + cmd.append(criteria) + solvables = __zypper__.nolock.noraise.xml.call(*cmd).getElementsByTagName('solvable') if not solvables: raise CommandExecutionError( 'No packages found matching \'{0}\''.format(criteria) ) out = {} - for solvable in [slv for slv in solvables - if slv.getAttribute('status') == 'not-installed' - and slv.getAttribute('kind') == 'package']: - out[solvable.getAttribute('name')] = {'summary': solvable.getAttribute('summary')} + for solvable in solvables: + out[solvable.getAttribute('name')] = dict() + for k,v in solvable.attributes.items(): + out[solvable.getAttribute('name')][k] = v return out From 3d28be0938dcf805ae19bffc3382300eb122e769 Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Tue, 14 Nov 2017 14:41:31 +0100 Subject: [PATCH 142/159] implement resolve_capabilities option for packages --- salt/modules/zypper.py | 98 +++++++++++++++++++++++++++++++++++++++++- salt/states/pkg.py | 88 ++++++++++++++++++++++++++++++++++++- 2 files changed, 183 insertions(+), 3 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index 6c717b9818c..e4c78769439 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -20,6 +20,7 @@ import re import os import time import datetime +import copy # Import 3rd-party libs # pylint: disable=import-error,redefined-builtin,no-name-in-module @@ -1049,6 +1050,10 @@ def install(name=None, operator (<, >, <=, >=, =) and a version number (ex. '>1.2.3-4'). This parameter is ignored if ``pkgs`` or ``sources`` is passed. + resolve_capabilities + If this option is set to True zypper will take capabilites into + account. In this case names which are just provided by a package + will get installed. Default is False. Multiple Package Installation Options: @@ -1164,7 +1169,13 @@ def install(name=None, log.info('Targeting repo \'{0}\''.format(fromrepo)) else: fromrepoopt = '' - cmd_install = ['install', '--name', '--auto-agree-with-licenses'] + cmd_install = ['install', '--auto-agree-with-licenses'] + + if kwargs.get('resolve_capabilities', False): + cmd_install.append('--capability') + else: + cmd_install.append('--name') + if not refresh: cmd_install.insert(0, '--no-refresh') if skip_verify: @@ -1195,6 +1206,7 @@ def install(name=None, __zypper__(no_repo_failure=ignore_repo_failure).call(*cmd) __context__.pop('pkg.list_pkgs', None) + __context__.pop('pkg.list_list_provides', None) new = list_pkgs(attr=diff_attr) if not downloadonly else list_downloaded() # Handle packages which report multiple new versions @@ -1312,6 +1324,7 @@ def upgrade(refresh=True, __zypper__(systemd_scope=_systemd_scope()).noraise.call(*cmd_update) __context__.pop('pkg.list_pkgs', None) + __context__.pop('pkg.list_list_provides', None) new = list_pkgs() # Handle packages which report multiple new versions @@ -1361,6 +1374,7 @@ def _uninstall(name=None, pkgs=None): targets = targets[500:] __context__.pop('pkg.list_pkgs', None) + __context__.pop('pkg.list_list_provides', None) ret = salt.utils.data.compare_dicts(old, list_pkgs()) if errors: @@ -2109,3 +2123,85 @@ def list_installed_patches(): salt '*' pkg.list_installed_patches ''' return _get_patches(installed_only=True) + +def list_provides(**kwargs): + ''' + List package provides currently installed as a dict. + {'': [' 1: + log.warn("Found ambiguous match for capability '{0}'.".format(pkg)) + + if version: + ret.append({name: version}) + else: + ret.append(name) + return ret diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 3ca512f6762..ea613988281 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -508,8 +508,11 @@ def _find_install_targets(name=None, # add it to the kwargs. kwargs['refresh'] = refresh + + resolve_capabilities = kwargs.get('resolve_capabilities', False) and 'pkg.list_provides' in __salt__ try: cur_pkgs = __salt__['pkg.list_pkgs'](versions_as_list=True, **kwargs) + cur_prov = resolve_capabilities and __salt__['pkg.list_provides'](**kwargs) or dict() except CommandExecutionError as exc: return {'name': name, 'changes': {}, @@ -669,6 +672,9 @@ def _find_install_targets(name=None, failed_verify = False for key, val in six.iteritems(desired): cver = cur_pkgs.get(key, []) + if resolve_capabilities and not cver and key in cur_prov: + cver = cur_pkgs.get(cur_prov.get(key)[0], []) + # Package not yet installed, so add to targets if not cver: targets[key] = val @@ -786,7 +792,7 @@ def _find_install_targets(name=None, warnings, was_refreshed) -def _verify_install(desired, new_pkgs, ignore_epoch=False): +def _verify_install(desired, new_pkgs, ignore_epoch=False, new_caps={}): ''' Determine whether or not the installed packages match what was requested in the SLS file. @@ -809,6 +815,8 @@ def _verify_install(desired, new_pkgs, ignore_epoch=False): cver = new_pkgs.get(pkgname.split('=')[0]) else: cver = new_pkgs.get(pkgname) + if not cver and pkgname in new_caps: + cver = new_pkgs.get(new_caps.get(pkgname)[0]) if not cver: failed.append(pkgname) @@ -872,6 +880,27 @@ def _nested_output(obj): ret = nested.output(obj).rstrip() return ret +def _resolve_capabilities(pkgs, refresh=False, **kwargs): + ''' + Resolve capabilities in ``pkgs`` and exchange them with real package + names, when the result is distinct. + This feature can be turned on while setting the paramter + ``resolve_capabilities`` to True. + + Return the input dictionary with replaced capability names and as + second return value a bool which indicate if a refresh was run. + + In case of ``resolve_capabilities`` is False (disabled) or not + supported by the implementation the input is returned unchanged. + ''' + was_refreshed = False + if not pkgs or 'pkg.resolve_capabilities' not in __salt__: + return (pkgs, was_refreshed) + + ret = __salt__['pkg.resolve_capabilities'](pkgs, refresh=refresh, **kwargs) + was_refreshed = refresh + return (ret, was_refreshed) + def installed( name, @@ -1105,6 +1134,11 @@ def installed( .. versionadded:: 2014.1.1 + :param bool resolve_capabilities: + Turn on resolving capabilities. This allow to name "provides" or alias names for packages. + + .. versionadded:: Oxygen + :param bool allow_updates: Allow the package to be updated outside Salt's control (e.g. auto updates on Windows). This means a package on the Minion can have a @@ -1448,6 +1482,14 @@ def installed( kwargs['saltenv'] = __env__ refresh = salt.utils.pkg.check_refresh(__opts__, refresh) + + # check if capabilities should be checked and modify the requested packages + # accordingly. + if pkgs: + (pkgs, was_refreshed) = _resolve_capabilities(pkgs, refresh=refresh, **kwargs) + if was_refreshed: + refresh = False + if not isinstance(pkg_verify, list): pkg_verify = pkg_verify is True if (pkg_verify or isinstance(pkg_verify, list)) \ @@ -1707,8 +1749,13 @@ def installed( if __grains__['os'] == 'FreeBSD': kwargs['with_origin'] = True new_pkgs = __salt__['pkg.list_pkgs'](versions_as_list=True, **kwargs) + if kwargs.get('resolve_capabilities', False) and 'pkg.list_provides' in __salt__: + new_caps = __salt__['pkg.list_provides'](**kwargs) + else: + new_caps = {} ok, failed = _verify_install(desired, new_pkgs, - ignore_epoch=ignore_epoch) + ignore_epoch=ignore_epoch, + new_caps=new_caps) modified = [x for x in ok if x in targets] not_modified = [x for x in ok if x not in targets @@ -1927,6 +1974,11 @@ def downloaded(name, - dos2unix - salt-minion: 2015.8.5-1.el6 + :param bool resolve_capabilities: + Turn on resolving capabilities. This allow to name "provides" or alias names for packages. + + .. versionadded:: Oxygen + CLI Example: .. code-block:: yaml @@ -1952,11 +2004,24 @@ def downloaded(name, ret['comment'] = 'No packages to download provided' return ret + # If just a name (and optionally a version) is passed, just pack them into + # the pkgs argument. + if name and not pkgs: + if version: + pkgs = [{name: version}] + version = None + else: + pkgs = [name] + # It doesn't make sense here to received 'downloadonly' as kwargs # as we're explicitely passing 'downloadonly=True' to execution module. if 'downloadonly' in kwargs: del kwargs['downloadonly'] + (pkgs, was_refreshed) = _resolve_capabilities(pkgs, **kwargs) + if was_refreshed: + refresh = False + # Only downloading not yet downloaded packages targets = _find_download_targets(name, version, @@ -2203,6 +2268,10 @@ def latest( This parameter is available only on Debian based distributions and has no effect on the rest. + :param bool resolve_capabilities: + Turn on resolving capabilities. This allow to name "provides" or alias names for packages. + + .. versionadded:: Oxygen Multiple Package Installation Options: @@ -2300,6 +2369,12 @@ def latest( kwargs['saltenv'] = __env__ + # check if capabilities should be checked and modify the requested packages + # accordingly. + (desired_pkgs, was_refreshed) = _resolve_capabilities(desired_pkgs, refresh=refresh, **kwargs) + if was_refreshed: + refresh = False + try: avail = __salt__['pkg.latest_version'](*desired_pkgs, fromrepo=fromrepo, @@ -2822,6 +2897,11 @@ def uptodate(name, refresh=False, pkgs=None, **kwargs): This parameter available only on Debian based distributions, and have no effect on the rest. + :param bool resolve_capabilities: + Turn on resolving capabilities. This allow to name "provides" or alias names for packages. + + .. versionadded:: Oxygen + kwargs Any keyword arguments to pass through to ``pkg.upgrade``. @@ -2841,7 +2921,11 @@ def uptodate(name, refresh=False, pkgs=None, **kwargs): ret['comment'] = '\'fromrepo\' argument not supported on this platform' return ret + if isinstance(refresh, bool): + (pkgs, was_refreshed) = _resolve_capabilities(pkgs, refresh=refresh, **kwargs) + if was_refreshed: + refresh = False try: packages = __salt__['pkg.list_upgrades'](refresh=refresh, **kwargs) if isinstance(pkgs, list): From 3d04f7d1fe9e107667a80e5d5dbae6f9073be70b Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Mon, 13 Nov 2017 14:12:26 +0100 Subject: [PATCH 143/159] test package capability feature --- tests/integration/states/test_pkg.py | 265 ++++++++++++++++++++++++++- 1 file changed, 264 insertions(+), 1 deletion(-) diff --git a/tests/integration/states/test_pkg.py b/tests/integration/states/test_pkg.py index 8406f6d6313..54129162523 100644 --- a/tests/integration/states/test_pkg.py +++ b/tests/integration/states/test_pkg.py @@ -37,11 +37,15 @@ _PKG_TARGETS = { 'Debian': ['python-plist', 'apg'], 'RedHat': ['units', 'zsh-html'], 'FreeBSD': ['aalib', 'pth'], - 'Suse': ['aalib', 'python-pssh'], + 'Suse': ['aalib', 'rpm-python'], 'MacOS': ['libpng', 'jpeg'], 'Windows': ['firefox', '7zip'], } +_PKG_CAP_TARGETS = { + 'Suse': [('w3m_ssl', 'w3m')], +} + _PKG_TARGETS_32 = { 'CentOS': 'xz-devel.i686' } @@ -793,3 +797,262 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): self.assertEqual(ret_comment, 'An error was encountered while installing/updating group ' '\'handle_missing_pkg_group\': Group \'handle_missing_pkg_group\' ' 'not found.') + + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') + @requires_system_grains + def test_pkg_cap_001_installed(self, grains=None): + ''' + This is a destructive test as it installs and then removes a package + ''' + # Skip test if package manager not available + if not pkgmgr_avail(self.run_function, self.run_function('grains.items')): + self.skipTest('Package manager is not available') + + os_family = grains.get('os_family', '') + pkg_cap_targets = _PKG_CAP_TARGETS.get(os_family, []) + if not len(pkg_cap_targets) > 0: + self.skipTest('Capability not provided') + + target, realpkg = pkg_cap_targets[0] + version = self.run_function('pkg.version', [target]) + realver = self.run_function('pkg.version', [realpkg]) + + # If this assert fails, we need to find new targets, this test needs to + # be able to test successful installation of packages, so this package + # needs to not be installed before we run the states below + self.assertFalse(version) + self.assertFalse(realver) + + ret = self.run_state('pkg.installed', name=target, refresh=False, resolve_capabilities=True, test=True) + self.assertInSaltComment("The following packages would be installed/updated: {0}".format(realpkg), ret) + ret = self.run_state('pkg.installed', name=target, refresh=False, resolve_capabilities=True) + self.assertSaltTrueReturn(ret) + ret = self.run_state('pkg.removed', name=realpkg) + self.assertSaltTrueReturn(ret) + + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') + @requires_system_grains + def test_pkg_cap_002_already_installed(self, grains=None): + ''' + This is a destructive test as it installs and then removes a package + ''' + # Skip test if package manager not available + if not pkgmgr_avail(self.run_function, self.run_function('grains.items')): + self.skipTest('Package manager is not available') + + os_family = grains.get('os_family', '') + pkg_cap_targets = _PKG_CAP_TARGETS.get(os_family, []) + if not len(pkg_cap_targets) > 0: + self.skipTest('Capability not provided') + + target, realpkg = pkg_cap_targets[0] + version = self.run_function('pkg.version', [target]) + realver = self.run_function('pkg.version', [realpkg]) + + # If this assert fails, we need to find new targets, this test needs to + # be able to test successful installation of packages, so this package + # needs to not be installed before we run the states below + self.assertFalse(version) + self.assertFalse(realver) + + # install the package already + ret = self.run_state('pkg.installed', name=realpkg, refresh=False) + + ret = self.run_state('pkg.installed', name=target, refresh=False, resolve_capabilities=True, test=True) + self.assertInSaltComment("All specified packages are already installed", ret) + + ret = self.run_state('pkg.installed', name=target, refresh=False, resolve_capabilities=True) + self.assertSaltTrueReturn(ret) + + self.assertInSaltComment("packages are already installed", ret) + ret = self.run_state('pkg.removed', name=realpkg) + self.assertSaltTrueReturn(ret) + + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') + @requires_system_grains + def test_pkg_cap_003_installed_multipkg_with_version(self, grains=None): + ''' + This is a destructive test as it installs and then removes two packages + ''' + # Skip test if package manager not available + if not pkgmgr_avail(self.run_function, self.run_function('grains.items')): + self.skipTest('Package manager is not available') + + os_family = grains.get('os_family', '') + pkg_cap_targets = _PKG_CAP_TARGETS.get(os_family, []) + if not len(pkg_cap_targets) > 0: + self.skipTest('Capability not provided') + pkg_targets = _PKG_TARGETS.get(os_family, []) + + # Don't perform this test on FreeBSD since version specification is not + # supported. + if os_family == 'FreeBSD': + return + + # Make sure that we have targets that match the os_family. If this + # fails then the _PKG_TARGETS dict above needs to have an entry added, + # with two packages that are not installed before these tests are run + self.assertTrue(bool(pkg_cap_targets)) + self.assertTrue(bool(pkg_targets)) + + if os_family == 'Arch': + for idx in range(13): + if idx == 12: + raise Exception('Package database locked after 60 seconds, ' + 'bailing out') + if not os.path.isfile('/var/lib/pacman/db.lck'): + break + time.sleep(5) + + capability, realpkg = pkg_cap_targets[0] + version = latest_version(self.run_function, pkg_targets[0]) + realver = latest_version(self.run_function, realpkg) + + # If this assert fails, we need to find new targets, this test needs to + # be able to test successful installation of packages, so these + # packages need to not be installed before we run the states below + self.assertTrue(bool(version)) + self.assertTrue(bool(realver)) + + pkgs = [{pkg_targets[0]: version}, pkg_targets[1], {capability: realver}] + ret = self.run_state('pkg.installed', + name='test_pkg_cap_003_installed_multipkg_with_version-install', + pkgs=pkgs, + refresh=False) + self.assertSaltFalseReturn(ret) + + ret = self.run_state('pkg.installed', + name='test_pkg_cap_003_installed_multipkg_with_version-install-capability', + pkgs=pkgs, + refresh=False, resolve_capabilities=True, test=True) + self.assertInSaltComment("packages would be installed/updated", ret) + self.assertInSaltComment("{0}={1}".format(realpkg, realver), ret) + + ret = self.run_state('pkg.installed', + name='test_pkg_cap_003_installed_multipkg_with_version-install-capability', + pkgs=pkgs, + refresh=False, resolve_capabilities=True) + self.assertSaltTrueReturn(ret) + cleanup_pkgs = pkg_targets + cleanup_pkgs.append(realpkg) + ret = self.run_state('pkg.removed', + name='test_pkg_cap_003_installed_multipkg_with_version-remove', + pkgs=cleanup_pkgs) + self.assertSaltTrueReturn(ret) + + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') + @requires_system_grains + def test_pkg_cap_004_latest(self, grains=None): + ''' + This tests pkg.latest with a package that has no epoch (or a zero + epoch). + ''' + # Skip test if package manager not available + if not pkgmgr_avail(self.run_function, self.run_function('grains.items')): + self.skipTest('Package manager is not available') + + os_family = grains.get('os_family', '') + pkg_cap_targets = _PKG_CAP_TARGETS.get(os_family, []) + if not len(pkg_cap_targets) > 0: + self.skipTest('Capability not provided') + + target, realpkg = pkg_cap_targets[0] + version = self.run_function('pkg.version', [target]) + realver = self.run_function('pkg.version', [realpkg]) + + # If this assert fails, we need to find new targets, this test needs to + # be able to test successful installation of packages, so this package + # needs to not be installed before we run the states below + self.assertFalse(version) + self.assertFalse(realver) + + ret = self.run_state('pkg.latest', name=target, refresh=False, resolve_capabilities=True, test=True) + self.assertInSaltComment("The following packages would be installed/upgraded: {0}".format(realpkg), ret) + ret = self.run_state('pkg.latest', name=target, refresh=False, resolve_capabilities=True) + self.assertSaltTrueReturn(ret) + + ret = self.run_state('pkg.latest', name=target, refresh=False, resolve_capabilities=True) + self.assertSaltTrueReturn(ret) + self.assertInSaltComment("is already up-to-date", ret) + + ret = self.run_state('pkg.removed', name=realpkg) + self.assertSaltTrueReturn(ret) + + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') + @requires_system_grains + def test_pkg_cap_005_downloaded(self, grains=None): + ''' + This is a destructive test as it installs and then removes a package + ''' + # Skip test if package manager not available + if not pkgmgr_avail(self.run_function, self.run_function('grains.items')): + self.skipTest('Package manager is not available') + + os_family = grains.get('os_family', '') + pkg_cap_targets = _PKG_CAP_TARGETS.get(os_family, []) + if not len(pkg_cap_targets) > 0: + self.skipTest('Capability not provided') + + target, realpkg = pkg_cap_targets[0] + version = self.run_function('pkg.version', [target]) + realver = self.run_function('pkg.version', [realpkg]) + + # If this assert fails, we need to find new targets, this test needs to + # be able to test successful installation of packages, so this package + # needs to not be installed before we run the states below + self.assertFalse(version) + self.assertFalse(realver) + + ret = self.run_state('pkg.downloaded', name=target, refresh=False) + self.assertSaltFalseReturn(ret) + + ret = self.run_state('pkg.downloaded', name=target, refresh=False, resolve_capabilities=True, test=True) + self.assertInSaltComment("The following packages would be downloaded: {0}".format(realpkg), ret) + + ret = self.run_state('pkg.downloaded', name=target, refresh=False, resolve_capabilities=True) + self.assertSaltTrueReturn(ret) + + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') + @requires_system_grains + def test_pkg_cap_006_uptodate(self, grains=None): + ''' + This is a destructive test as it installs and then removes a package + ''' + # Skip test if package manager not available + if not pkgmgr_avail(self.run_function, self.run_function('grains.items')): + self.skipTest('Package manager is not available') + + os_family = grains.get('os_family', '') + pkg_cap_targets = _PKG_CAP_TARGETS.get(os_family, []) + if not len(pkg_cap_targets) > 0: + self.skipTest('Capability not provided') + + target, realpkg = pkg_cap_targets[0] + version = self.run_function('pkg.version', [target]) + realver = self.run_function('pkg.version', [realpkg]) + + # If this assert fails, we need to find new targets, this test needs to + # be able to test successful installation of packages, so this package + # needs to not be installed before we run the states below + self.assertFalse(version) + self.assertFalse(realver) + + ret = self.run_state('pkg.installed', name=target, + refresh=False, resolve_capabilities=True) + self.assertSaltTrueReturn(ret) + ret = self.run_state('pkg.uptodate', + name='test_pkg_cap_006_uptodate', + pkgs=[target], + refresh=False, + resolve_capabilities=True) + self.assertSaltTrueReturn(ret) + self.assertInSaltComment("System is already up-to-date", ret) + ret = self.run_state('pkg.removed', name=realpkg) + self.assertSaltTrueReturn(ret) + ret = self.run_state('pkg.uptodate', + name='test_pkg_cap_006_uptodate', + refresh=False, + test=True) + self.assertInSaltComment("System update will be performed", ret) + + From 8433491caeabed3a42a9486a2a4960b103c4a1d9 Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Thu, 16 Nov 2017 09:02:06 +0100 Subject: [PATCH 144/159] fix list_provides cache handling --- salt/modules/zypper.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index e4c78769439..ff38ebc809b 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -1206,7 +1206,7 @@ def install(name=None, __zypper__(no_repo_failure=ignore_repo_failure).call(*cmd) __context__.pop('pkg.list_pkgs', None) - __context__.pop('pkg.list_list_provides', None) + __context__.pop('pkg.list_provides', None) new = list_pkgs(attr=diff_attr) if not downloadonly else list_downloaded() # Handle packages which report multiple new versions @@ -1324,7 +1324,7 @@ def upgrade(refresh=True, __zypper__(systemd_scope=_systemd_scope()).noraise.call(*cmd_update) __context__.pop('pkg.list_pkgs', None) - __context__.pop('pkg.list_list_provides', None) + __context__.pop('pkg.list_provides', None) new = list_pkgs() # Handle packages which report multiple new versions @@ -1374,7 +1374,7 @@ def _uninstall(name=None, pkgs=None): targets = targets[500:] __context__.pop('pkg.list_pkgs', None) - __context__.pop('pkg.list_list_provides', None) + __context__.pop('pkg.list_provides', None) ret = salt.utils.data.compare_dicts(old, list_pkgs()) if errors: @@ -2129,22 +2129,20 @@ def list_provides(**kwargs): List package provides currently installed as a dict. {'': [' Date: Thu, 16 Nov 2017 09:02:38 +0100 Subject: [PATCH 145/159] improve search function --- salt/modules/zypper.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index ff38ebc809b..e8621b27583 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -1823,7 +1823,7 @@ def search(criteria, refresh=False, **kwargs): if refresh: refresh_db() - cmd = ['se'] + cmd = ['search'] if kwargs.get('match') == 'exact': cmd.append('--match-exact') elif kwargs.get('match') == 'words': @@ -1853,7 +1853,8 @@ def search(criteria, refresh=False, **kwargs): if kwargs.get('installed_only'): cmd.append('--installed-only') if kwargs.get('not_installed_only'): - cmd.append('--not-installed-only') + # long parameter was renamed in the past + cmd.append('-u') if kwargs.get('details'): cmd.append('--details') From 06b2d25371fa5e5ec78f6effa49779b36ebd505e Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Thu, 16 Nov 2017 09:03:01 +0100 Subject: [PATCH 146/159] simplify argument handling --- salt/modules/zypper.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index e8621b27583..81507428362 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -1171,10 +1171,7 @@ def install(name=None, fromrepoopt = '' cmd_install = ['install', '--auto-agree-with-licenses'] - if kwargs.get('resolve_capabilities', False): - cmd_install.append('--capability') - else: - cmd_install.append('--name') + cmd_install.append(kwargs.get('resolve_capabilities') and '--capability' or '--name') if not refresh: cmd_install.insert(0, '--no-refresh') From ab9f5768b71d008a1b1b857f75c1f9da78cc72c5 Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Thu, 16 Nov 2017 09:10:11 +0100 Subject: [PATCH 147/159] remove unused variable and simplify dict size checking --- salt/modules/zypper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index 81507428362..dbb26c0ff28 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -2187,13 +2187,13 @@ def resolve_capabilities(pkgs, refresh, **kwargs): if kwargs.get('resolve_capabilities', False): try: search(name, match='exact') - except CommandExecutionError as e: + except CommandExecutionError: # no package this such a name found # search for a package which provides this name result = search(name, provides=True, match='exact') - if len(result.keys()) == 1: + if len(result) == 1: name = result.keys()[0] - elif len(result.keys()) > 1: + elif len(result) > 1: log.warn("Found ambiguous match for capability '{0}'.".format(pkg)) if version: From e0764a7362d26be8a09e39d6d384ae72d1f11ac9 Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Thu, 16 Nov 2017 09:12:04 +0100 Subject: [PATCH 148/159] catch search exceptions --- salt/modules/zypper.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index dbb26c0ff28..4a23afc14b9 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -2190,11 +2190,15 @@ def resolve_capabilities(pkgs, refresh, **kwargs): except CommandExecutionError: # no package this such a name found # search for a package which provides this name - result = search(name, provides=True, match='exact') - if len(result) == 1: - name = result.keys()[0] - elif len(result) > 1: - log.warn("Found ambiguous match for capability '{0}'.".format(pkg)) + try: + result = search(name, provides=True, match='exact') + if len(result) == 1: + name = result.keys()[0] + elif len(result) > 1: + log.warn("Found ambiguous match for capability '{0}'.".format(pkg)) + except CommandExecutionError: + # when search throw an exception stay with original name and version + pass if version: ret.append({name: version}) From facbf90186f5a5d24cbc85fa48a8bb34272392b0 Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Thu, 16 Nov 2017 09:25:21 +0100 Subject: [PATCH 149/159] remove extra parenthesis --- salt/states/pkg.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index ea613988281..a3a704088fc 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -1486,7 +1486,7 @@ def installed( # check if capabilities should be checked and modify the requested packages # accordingly. if pkgs: - (pkgs, was_refreshed) = _resolve_capabilities(pkgs, refresh=refresh, **kwargs) + pkgs, was_refreshed = _resolve_capabilities(pkgs, refresh=refresh, **kwargs) if was_refreshed: refresh = False @@ -2018,7 +2018,7 @@ def downloaded(name, if 'downloadonly' in kwargs: del kwargs['downloadonly'] - (pkgs, was_refreshed) = _resolve_capabilities(pkgs, **kwargs) + pkgs, was_refreshed = _resolve_capabilities(pkgs, **kwargs) if was_refreshed: refresh = False @@ -2371,7 +2371,7 @@ def latest( # check if capabilities should be checked and modify the requested packages # accordingly. - (desired_pkgs, was_refreshed) = _resolve_capabilities(desired_pkgs, refresh=refresh, **kwargs) + desired_pkgs, was_refreshed = _resolve_capabilities(desired_pkgs, refresh=refresh, **kwargs) if was_refreshed: refresh = False @@ -2923,7 +2923,7 @@ def uptodate(name, refresh=False, pkgs=None, **kwargs): if isinstance(refresh, bool): - (pkgs, was_refreshed) = _resolve_capabilities(pkgs, refresh=refresh, **kwargs) + pkgs, was_refreshed = _resolve_capabilities(pkgs, refresh=refresh, **kwargs) if was_refreshed: refresh = False try: From c77afffe485c523fbf58735b6ba33836d8a251f2 Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Thu, 16 Nov 2017 09:36:57 +0100 Subject: [PATCH 150/159] simplify refresh handling --- salt/states/pkg.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index a3a704088fc..730610daefc 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -888,18 +888,16 @@ def _resolve_capabilities(pkgs, refresh=False, **kwargs): ``resolve_capabilities`` to True. Return the input dictionary with replaced capability names and as - second return value a bool which indicate if a refresh was run. + second return value a bool which say if a refresh need to be run. In case of ``resolve_capabilities`` is False (disabled) or not supported by the implementation the input is returned unchanged. ''' - was_refreshed = False if not pkgs or 'pkg.resolve_capabilities' not in __salt__: - return (pkgs, was_refreshed) + return pkgs, refresh ret = __salt__['pkg.resolve_capabilities'](pkgs, refresh=refresh, **kwargs) - was_refreshed = refresh - return (ret, was_refreshed) + return ret, False def installed( @@ -1486,9 +1484,7 @@ def installed( # check if capabilities should be checked and modify the requested packages # accordingly. if pkgs: - pkgs, was_refreshed = _resolve_capabilities(pkgs, refresh=refresh, **kwargs) - if was_refreshed: - refresh = False + pkgs, refresh = _resolve_capabilities(pkgs, refresh=refresh, **kwargs) if not isinstance(pkg_verify, list): pkg_verify = pkg_verify is True @@ -2018,9 +2014,7 @@ def downloaded(name, if 'downloadonly' in kwargs: del kwargs['downloadonly'] - pkgs, was_refreshed = _resolve_capabilities(pkgs, **kwargs) - if was_refreshed: - refresh = False + pkgs, _refresh = _resolve_capabilities(pkgs, **kwargs) # Only downloading not yet downloaded packages targets = _find_download_targets(name, @@ -2371,9 +2365,7 @@ def latest( # check if capabilities should be checked and modify the requested packages # accordingly. - desired_pkgs, was_refreshed = _resolve_capabilities(desired_pkgs, refresh=refresh, **kwargs) - if was_refreshed: - refresh = False + desired_pkgs, refresh = _resolve_capabilities(desired_pkgs, refresh=refresh, **kwargs) try: avail = __salt__['pkg.latest_version'](*desired_pkgs, @@ -2923,7 +2915,7 @@ def uptodate(name, refresh=False, pkgs=None, **kwargs): if isinstance(refresh, bool): - pkgs, was_refreshed = _resolve_capabilities(pkgs, refresh=refresh, **kwargs) + pkgs, refresh = _resolve_capabilities(pkgs, refresh=refresh, **kwargs) if was_refreshed: refresh = False try: From 966d85d39386256f5140b63678a5c610af48d166 Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Thu, 16 Nov 2017 09:59:06 +0100 Subject: [PATCH 151/159] better way to get key/value from a one item dict --- salt/modules/zypper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index 4a23afc14b9..a02d238966c 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -2178,8 +2178,8 @@ def resolve_capabilities(pkgs, refresh, **kwargs): ret = list() for pkg in pkgs: if isinstance(pkg, dict): - for name, version in pkg.items(): - break + name = next(iter(pkg)) + version = pkg[name] else: name = pkg version = None From b6365aee3c46d37a9409f222439a025561b4ea1f Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Thu, 16 Nov 2017 13:13:11 +0100 Subject: [PATCH 152/159] fix order of parameters in unit tests --- tests/unit/modules/test_zypper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/modules/test_zypper.py b/tests/unit/modules/test_zypper.py index 37852e155e0..e68f8a4d9fa 100644 --- a/tests/unit/modules/test_zypper.py +++ b/tests/unit/modules/test_zypper.py @@ -669,8 +669,8 @@ Repository 'DUMMY' not found by its alias, number, or URI. zypper_mock.assert_called_once_with( '--no-refresh', 'install', - '--name', '--auto-agree-with-licenses', + '--name', '--download-only', 'vim' ) @@ -699,8 +699,8 @@ Repository 'DUMMY' not found by its alias, number, or URI. zypper_mock.assert_called_once_with( '--no-refresh', 'install', - '--name', '--auto-agree-with-licenses', + '--name', '--download-only', 'vim' ) @@ -724,8 +724,8 @@ Repository 'DUMMY' not found by its alias, number, or URI. zypper_mock.assert_called_once_with( '--no-refresh', 'install', - '--name', '--auto-agree-with-licenses', + '--name', 'patch:SUSE-PATCH-1234' ) self.assertDictEqual(ret, {"vim": {"old": "1.1", "new": "1.2"}}) From f3f1abe5ff8da00245be199f4ece3352c6248453 Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Thu, 16 Nov 2017 14:25:19 +0100 Subject: [PATCH 153/159] pylint fixes --- salt/modules/zypper.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index a02d238966c..9cb767da178 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -20,7 +20,6 @@ import re import os import time import datetime -import copy # Import 3rd-party libs # pylint: disable=import-error,redefined-builtin,no-name-in-module @@ -1865,7 +1864,7 @@ def search(criteria, refresh=False, **kwargs): out = {} for solvable in solvables: out[solvable.getAttribute('name')] = dict() - for k,v in solvable.attributes.items(): + for k, v in solvable.attributes.items(): out[solvable.getAttribute('name')][k] = v return out @@ -2122,6 +2121,7 @@ def list_installed_patches(): ''' return _get_patches(installed_only=True) + def list_provides(**kwargs): ''' List package provides currently installed as a dict. @@ -2144,6 +2144,7 @@ def list_provides(**kwargs): return ret + def resolve_capabilities(pkgs, refresh, **kwargs): ''' .. versionadded:: Oxygen From b6e60fa7a86df5c10355f15cdcf109e16ddecf1b Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Thu, 16 Nov 2017 14:30:53 +0100 Subject: [PATCH 154/159] write function to clean caches --- salt/modules/zypper.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index 9cb767da178..ff77042dcda 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -399,6 +399,14 @@ def _systemd_scope(): and __salt__['config.get']('systemd.scope', True) +def _clean_cache(): + ''' + Clean cached results + ''' + for cache_name in ['list_pkgs', 'list_provides']: + __context__.pop(cache_name, None) + + def list_upgrades(refresh=True, **kwargs): ''' List all available package upgrades on this system @@ -1201,8 +1209,7 @@ def install(name=None, downgrades = downgrades[500:] __zypper__(no_repo_failure=ignore_repo_failure).call(*cmd) - __context__.pop('pkg.list_pkgs', None) - __context__.pop('pkg.list_provides', None) + _clean_cache() new = list_pkgs(attr=diff_attr) if not downloadonly else list_downloaded() # Handle packages which report multiple new versions @@ -1319,8 +1326,7 @@ def upgrade(refresh=True, old = list_pkgs() __zypper__(systemd_scope=_systemd_scope()).noraise.call(*cmd_update) - __context__.pop('pkg.list_pkgs', None) - __context__.pop('pkg.list_provides', None) + _clean_cache() new = list_pkgs() # Handle packages which report multiple new versions @@ -1369,8 +1375,7 @@ def _uninstall(name=None, pkgs=None): __zypper__(systemd_scope=systemd_scope).call('remove', *targets[:500]) targets = targets[500:] - __context__.pop('pkg.list_pkgs', None) - __context__.pop('pkg.list_provides', None) + _clean_cache() ret = salt.utils.data.compare_dicts(old, list_pkgs()) if errors: From 55553bab96cb125bd06ff8f2317371917e958b75 Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Thu, 16 Nov 2017 14:33:52 +0100 Subject: [PATCH 155/159] log debug message when ignoring exceptions --- salt/modules/zypper.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index ff77042dcda..42afb1f799d 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -2202,8 +2202,9 @@ def resolve_capabilities(pkgs, refresh, **kwargs): name = result.keys()[0] elif len(result) > 1: log.warn("Found ambiguous match for capability '{0}'.".format(pkg)) - except CommandExecutionError: - # when search throw an exception stay with original name and version + except CommandExecutionError as exc: + # when search throws an exception stay with original name and version + log.debug("Search failed with: {0}".format(exc)) pass if version: From 1cdd741dcfb3752001af89779455ac3733071933 Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Thu, 16 Nov 2017 14:36:44 +0100 Subject: [PATCH 156/159] fix cache name --- salt/modules/zypper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index 42afb1f799d..16e2b987df2 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -403,8 +403,8 @@ def _clean_cache(): ''' Clean cached results ''' - for cache_name in ['list_pkgs', 'list_provides']: - __context__.pop(cache_name, None) + for cache_name in ['pkg.list_pkgs', 'pkg.list_provides']: + __context__.pop(cache_name, None) def list_upgrades(refresh=True, **kwargs): From e42fe24dd7273eda33991f62d8c2f7f51fc9d7c6 Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Thu, 16 Nov 2017 15:10:24 +0100 Subject: [PATCH 157/159] use mapping for allowed search option --- salt/modules/zypper.py | 43 +++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index 16e2b987df2..c2017bc917f 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -1821,6 +1821,20 @@ def search(criteria, refresh=False, **kwargs): salt '*' pkg.search ''' + ALLOWED_SEARCH_OPTIONS = { + 'provides': '--provides', + 'recommends': '--recommends', + 'requires': '--requires', + 'suggests': '--suggests', + 'conflicts': '--conflicts', + 'obsoletes': '--obsoletes', + 'file_list': '--file-list', + 'search_descriptions': '--search-descriptions', + 'case_sensitive': '--case-sensitive', + 'installed_only': '--installed-only', + 'not_installed_only': '-u', + 'details': '--details' + } if refresh: refresh_db() @@ -1832,32 +1846,9 @@ def search(criteria, refresh=False, **kwargs): elif kwargs.get('match') == 'substrings': cmd.append('--match-substrings') - if kwargs.get('provides'): - cmd.append('--provides') - if kwargs.get('recommends'): - cmd.append('--recommends') - if kwargs.get('requires'): - cmd.append('--requires') - if kwargs.get('suggests'): - cmd.append('--suggests') - if kwargs.get('conflicts'): - cmd.append('--conflicts') - if kwargs.get('obsoletes'): - cmd.append('--obsoletes') - - if kwargs.get('file_list'): - cmd.append('--file-list') - if kwargs.get('search_descriptions'): - cmd.append('--search-descriptions') - if kwargs.get('case_sensitive'): - cmd.append('--case-sensitive') - if kwargs.get('installed_only'): - cmd.append('--installed-only') - if kwargs.get('not_installed_only'): - # long parameter was renamed in the past - cmd.append('-u') - if kwargs.get('details'): - cmd.append('--details') + for opt in kwargs: + if opt in ALLOWED_SEARCH_OPTIONS: + cmd.append(ALLOWED_SEARCH_OPTIONS.get(opt)) cmd.append(criteria) solvables = __zypper__.nolock.noraise.xml.call(*cmd).getElementsByTagName('solvable') From bbf542d5d156419a971bdfcbb07c2df8a9ab9519 Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Thu, 16 Nov 2017 19:06:00 +0100 Subject: [PATCH 158/159] pylint fixes --- salt/modules/zypper.py | 1 - salt/states/pkg.py | 9 ++++----- tests/integration/states/test_pkg.py | 2 -- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index c2017bc917f..b5437ca6137 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -2196,7 +2196,6 @@ def resolve_capabilities(pkgs, refresh, **kwargs): except CommandExecutionError as exc: # when search throws an exception stay with original name and version log.debug("Search failed with: {0}".format(exc)) - pass if version: ret.append({name: version}) diff --git a/salt/states/pkg.py b/salt/states/pkg.py index 730610daefc..d438cd82103 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -508,7 +508,6 @@ def _find_install_targets(name=None, # add it to the kwargs. kwargs['refresh'] = refresh - resolve_capabilities = kwargs.get('resolve_capabilities', False) and 'pkg.list_provides' in __salt__ try: cur_pkgs = __salt__['pkg.list_pkgs'](versions_as_list=True, **kwargs) @@ -792,13 +791,15 @@ def _find_install_targets(name=None, warnings, was_refreshed) -def _verify_install(desired, new_pkgs, ignore_epoch=False, new_caps={}): +def _verify_install(desired, new_pkgs, ignore_epoch=False, new_caps=None): ''' Determine whether or not the installed packages match what was requested in the SLS file. ''' ok = [] failed = [] + if not new_caps: + new_caps = dict() for pkgname, pkgver in desired.items(): # FreeBSD pkg supports `openjdk` and `java/openjdk7` package names. # Homebrew for Mac OSX does something similar with tap names @@ -880,6 +881,7 @@ def _nested_output(obj): ret = nested.output(obj).rstrip() return ret + def _resolve_capabilities(pkgs, refresh=False, **kwargs): ''' Resolve capabilities in ``pkgs`` and exchange them with real package @@ -2913,11 +2915,8 @@ def uptodate(name, refresh=False, pkgs=None, **kwargs): ret['comment'] = '\'fromrepo\' argument not supported on this platform' return ret - if isinstance(refresh, bool): pkgs, refresh = _resolve_capabilities(pkgs, refresh=refresh, **kwargs) - if was_refreshed: - refresh = False try: packages = __salt__['pkg.list_upgrades'](refresh=refresh, **kwargs) if isinstance(pkgs, list): diff --git a/tests/integration/states/test_pkg.py b/tests/integration/states/test_pkg.py index 54129162523..24243c424d7 100644 --- a/tests/integration/states/test_pkg.py +++ b/tests/integration/states/test_pkg.py @@ -1054,5 +1054,3 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): refresh=False, test=True) self.assertInSaltComment("System update will be performed", ret) - - From dbcd6d7311d615a20492cd99af683cc6fec427ff Mon Sep 17 00:00:00 2001 From: Michael Calmer Date: Tue, 21 Nov 2017 09:02:51 +0100 Subject: [PATCH 159/159] improve documentation of list_provides --- salt/modules/zypper.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index b5437ca6137..b831142c83c 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -2120,8 +2120,16 @@ def list_installed_patches(): def list_provides(**kwargs): ''' - List package provides currently installed as a dict. - {'': ['': ['', '', ...]} + + CLI Examples: + + .. code-block:: bash + + salt '*' pkg.list_provides ''' ret = __context__.get('pkg.list_provides') if not ret: