Merge pull request #46908 from rallytime/merge-2018.3

[2018.3] Merge forward from 2017.7 to 2018.3

salt/acl/__init__.py

@@ -41,3 +41,15 @@ class PublisherACL(object):
             if not salt.utils.stringutils.check_whitelist_blacklist(fun, blacklist=self.blacklist.get('modules', [])):
                 return True
         return False
+
+    def user_is_whitelisted(self, user):
+        return salt.utils.stringutils.check_whitelist_blacklist(user, whitelist=self.blacklist.get('users', []))
+
+    def cmd_is_whitelisted(self, cmd):
+        # If this is a regular command, it is a single function
+        if isinstance(cmd, str):
+            cmd = [cmd]
+        for fun in cmd:
+            if salt.utils.stringutils.check_whitelist_blacklist(fun, whitelist=self.blacklist.get('modules', [])):
+                return True
+        return False

salt/auth/pam.py

@@ -48,15 +48,20 @@ from salt.ext.six.moves import range  # pylint: disable=import-error,redefined-b
 # Import 3rd-party libs
 from salt.ext import six
 
-LIBC = CDLL(find_library('c'))
+try:
+    LIBC = CDLL(find_library('c'))
 
-CALLOC = LIBC.calloc
-CALLOC.restype = c_void_p
-CALLOC.argtypes = [c_uint, c_uint]
+    CALLOC = LIBC.calloc
+    CALLOC.restype = c_void_p
+    CALLOC.argtypes = [c_uint, c_uint]
 
-STRDUP = LIBC.strdup
-STRDUP.argstypes = [c_char_p]
-STRDUP.restype = POINTER(c_char)  # NOT c_char_p !!!!
+    STRDUP = LIBC.strdup
+    STRDUP.argstypes = [c_char_p]
+    STRDUP.restype = POINTER(c_char)  # NOT c_char_p !!!!
+except AttributeError:
+    HAS_LIBC = False
+else:
+    HAS_LIBC = True
 
 # Various constants
 PAM_PROMPT_ECHO_OFF = 1
@@ -147,7 +152,7 @@ def __virtual__():
     '''
     Only load on Linux systems
     '''
-    return HAS_PAM
+    return HAS_LIBC and HAS_PAM
 
 
 def authenticate(username, password):

salt/client/__init__.py

@@ -249,7 +249,7 @@ class LocalClient(object):
 
         return pub_data
 
-    def _check_pub_data(self, pub_data):
+    def _check_pub_data(self, pub_data, listen=True):
         '''
         Common checks on the pub_data data structure returned from running pub
         '''
@@ -282,7 +282,13 @@ class LocalClient(object):
                 print('No minions matched the target. '
                       'No command was sent, no jid was assigned.')
                 return {}
-        else:
+
+        # don't install event subscription listeners when the request is async
+        # and doesn't care. this is important as it will create event leaks otherwise
+        if not listen:
+            return pub_data
+
+        if self.opts.get('order_masters'):
             self.event.subscribe('syndic/.*/{0}'.format(pub_data['jid']), 'regex')
 
         self.event.subscribe('salt/job/{0}'.format(pub_data['jid']))
@@ -346,7 +352,7 @@ class LocalClient(object):
             # Convert to generic client error and pass along message
             raise SaltClientError(general_exception)
 
-        return self._check_pub_data(pub_data)
+        return self._check_pub_data(pub_data, listen=listen)
 
     def gather_minions(self, tgt, expr_form):
         _res = salt.utils.minions.CkMinions(self.opts).check_minions(tgt, tgt_type=expr_form)
@@ -412,7 +418,7 @@ class LocalClient(object):
             # Convert to generic client error and pass along message
             raise SaltClientError(general_exception)
 
-        raise tornado.gen.Return(self._check_pub_data(pub_data))
+        raise tornado.gen.Return(self._check_pub_data(pub_data, listen=listen))
 
     def cmd_async(
             self,
@@ -453,6 +459,7 @@ class LocalClient(object):
                                 tgt_type,
                                 ret,
                                 jid=jid,
+                                listen=False,
                                 **kwargs)
         try:
             return pub_data['jid']
@@ -478,6 +485,8 @@ class LocalClient(object):
         following exceptions.
 
         :param sub: The number of systems to execute on
+        :param cli: When this is set to True, a generator is returned,
+                    otherwise a dictionary of the minion returns is returned
 
         .. code-block:: python
 

salt/grains/core.py

@@ -805,19 +805,6 @@ def _virtual(osdata):
                     grains['virtual_subtype'] = 'chroot'
             except (IOError, OSError):
                 pass
-        if os.path.isfile('/proc/1/cgroup'):
-            try:
-                with salt.utils.files.fopen('/proc/1/cgroup', 'r') as fhr:
-                    fhr_contents = fhr.read()
-                if ':/lxc/' in fhr_contents:
-                    grains['virtual_subtype'] = 'LXC'
-                else:
-                    if any(x in fhr_contents
-                           for x in (':/system.slice/docker', ':/docker/',
-                                     ':/docker-ce/')):
-                        grains['virtual_subtype'] = 'Docker'
-            except IOError:
-                pass
         if isdir('/proc/vz'):
             if os.path.isfile('/proc/vz/version'):
                 grains['virtual'] = 'openvzhn'
@@ -867,6 +854,20 @@ def _virtual(osdata):
             # If a Dom0 or DomU was detected, obviously this is xen
             if 'dom' in grains.get('virtual_subtype', '').lower():
                 grains['virtual'] = 'xen'
+        # Check container type after hypervisors, to avoid variable overwrite on containers running in virtual environment.
+        if os.path.isfile('/proc/1/cgroup'):
+            try:
+                with salt.utils.files.fopen('/proc/1/cgroup', 'r') as fhr:
+                    fhr_contents = fhr.read()
+                if ':/lxc/' in fhr_contents:
+                    grains['virtual_subtype'] = 'LXC'
+                else:
+                    if any(x in fhr_contents
+                           for x in (':/system.slice/docker', ':/docker/',
+                                     ':/docker-ce/')):
+                        grains['virtual_subtype'] = 'Docker'
+            except IOError:
+                pass
         if os.path.isfile('/proc/cpuinfo'):
             with salt.utils.files.fopen('/proc/cpuinfo', 'r') as fhr:
                 if 'QEMU Virtual CPU' in fhr.read():

salt/modules/saltutil.py

@@ -1348,7 +1348,7 @@ def _exec(client, tgt, fun, arg, timeout, tgt_type, ret, kwarg, **kwargs):
         _cmd = client.cmd_subset
         cmd_kwargs = {
             'tgt': tgt, 'fun': fun, 'arg': arg, 'tgt_type': tgt_type,
-            'ret': ret, 'kwarg': kwarg, 'sub': kwargs['subset'],
+            'ret': ret, 'cli': True, 'kwarg': kwarg, 'sub': kwargs['subset'],
         }
         del kwargs['subset']
     else:

salt/modules/yumpkg.py

@@ -2711,7 +2711,8 @@ def mod_repo(repo, basedir=None, **kwargs):
 
     # Build a list of keys to be deleted
     todelete = []
-    for key in repo_opts:
+    # list() of keys because the dict could be shrinking in the for loop.
+    for key in list(repo_opts):
         if repo_opts[key] != 0 and not repo_opts[key]:
             del repo_opts[key]
             todelete.append(key)

salt/utils/data.py

@@ -568,7 +568,7 @@ def subdict_match(data,
                            exact_match=exact_match):
                 return True
             continue
-        if isinstance(match, list):
+        if isinstance(match, (list, tuple)):
             # We are matching a single component to a single list member
             for member in match:
                 if isinstance(member, dict):

salt/utils/win_dacl.py

@@ -1102,8 +1102,13 @@ def get_name(principal):
     Gets the name from the specified principal.
 
     Args:
-        principal (str): Find the Normalized name based on this. Can be a PySID
-        object, a SID string, or a user name in any capitalization.
+        principal (str):
+            Find the Normalized name based on this. Can be a PySID object, a SID
+            string, or a user name in any capitalization.
+
+            .. note::
+                Searching based on the user name can be slow on hosts connected
+                to large Active Directory domains.
 
     Returns:
         str: The name that corresponds to the passed principal
@@ -1115,34 +1120,35 @@ def get_name(principal):
         salt.utils.win_dacl.get_name('S-1-5-32-544')
         salt.utils.win_dacl.get_name('adminisTrators')
     '''
-    # If None is passed, use the Universal Well-known SID for "Null SID"
-    if principal is None:
-        principal = 'S-1-0-0'
-
-    # Assume PySID object
-    sid_obj = principal
-
-    # Name
-    try:
-        sid_obj = win32security.LookupAccountName(None, principal)[0]
-    except (pywintypes.error, TypeError):
-        pass
-
-    # String SID
-    try:
-        sid_obj = win32security.ConvertStringSidToSid(principal)
-    except (pywintypes.error, TypeError):
-        pass
+    # If this is a PySID object, use it
+    if isinstance(principal, pywintypes.SIDType):
+        sid_obj = principal
+    else:
+        # If None is passed, use the Universal Well-known SID for "Null SID"
+        if principal is None:
+            principal = 'S-1-0-0'
+        # Try Converting String SID to SID Object first as it's least expensive
+        try:
+            sid_obj = win32security.ConvertStringSidToSid(principal)
+        except pywintypes.error:
+            # Try Getting the SID Object by Name Lookup last
+            # This is expensive, especially on large AD Domains
+            try:
+                sid_obj = win32security.LookupAccountName(None, principal)[0]
+            except pywintypes.error:
+                # This is not a PySID object, a SID String, or a valid Account
+                # Name. Just pass it and let the LookupAccountSid function try
+                # to resolve it
+                sid_obj = principal
 
+    # By now we should have a valid PySID object
     try:
         return win32security.LookupAccountSid(None, sid_obj)[0]
     except (pywintypes.error, TypeError) as exc:
+        message = 'Error resolving "{0}"'.format(principal)
         if type(exc) == pywintypes.error:
             win_error = win32api.FormatMessage(exc.winerror).rstrip('\n')
-            message = 'Error resolving {0} ({1})'.format(principal, win_error)
-        else:
-            message = 'Error resolving {0}'.format(principal)
-
+            message = '{0}: {1}'.format(message, win_error)
         raise CommandExecutionError(message)
 
 

tests/integration/cloud/providers/test_ec2.py

@@ -220,10 +220,10 @@ class EC2Test(ShellCase):
         # TODO: winexe calls hang and the test fails by timing out. The same
         # same calls succeed when run outside of the test environment.
         self.override_profile_config(
-            'ec2-win2012-test',
+            'ec2-win2012r2-test',
             {
                 'use_winrm': False,
-                'user_data': self.copy_file('windows-firewall-winexe.ps1'),
+                'userdata_file': self.copy_file('windows-firewall-winexe.ps1'),
                 'win_installer': self.copy_file(self.INSTALLER),
             },
         )
@@ -235,9 +235,9 @@ class EC2Test(ShellCase):
         winrm (classic)
         '''
         self.override_profile_config(
-            'ec2-win2016-test',
+            'ec2-win2012r2-test',
             {
-                'user_data': self.copy_file('windows-firewall.ps1'),
+                'userdata_file': self.copy_file('windows-firewall.ps1'),
                 'win_installer': self.copy_file(self.INSTALLER),
                 'winrm_ssl_verify': False,
             }
@@ -257,7 +257,7 @@ class EC2Test(ShellCase):
             'ec2-win2016-test',
             {
                 'use_winrm': False,
-                'user_data': self.copy_file('windows-firewall-winexe.ps1'),
+                'userdata_file': self.copy_file('windows-firewall-winexe.ps1'),
                 'win_installer': self.copy_file(self.INSTALLER),
             },
         )
@@ -271,7 +271,7 @@ class EC2Test(ShellCase):
         self.override_profile_config(
             'ec2-win2016-test',
             {
-                'user_data': self.copy_file('windows-firewall.ps1'),
+                'userdata_file': self.copy_file('windows-firewall.ps1'),
                 'win_installer': self.copy_file(self.INSTALLER),
                 'winrm_ssl_verify': False,
             }

tests/integration/files/file/base/issue-46762.sls

@@ -0,0 +1,11 @@
+a: test.fail_without_changes
+
+b:
+  test.nop:
+    - prereq:
+      - c
+
+c:
+  test.nop:
+  - require:
+    - a

tests/integration/files/file/base/orch/subset.sls

@@ -0,0 +1,5 @@
+test subset:
+  salt.state:
+    - tgt: '*'
+    - subset: 1
+    - sls: test

tests/integration/files/file/base/test.sls

@@ -0,0 +1,3 @@
+test state:
+  test.succeed_without_changes:
+    - name: test

tests/integration/modules/test_grains.py

@@ -153,8 +153,7 @@ class GrainsAppendTestCase(ModuleCase):
     GRAIN_VAL = 'my-grain-val'
 
     def tearDown(self):
-        for item in self.run_function('grains.get', [self.GRAIN_KEY]):
-            self.run_function('grains.remove', [self.GRAIN_KEY, item])
+        self.run_function('grains.setval', [self.GRAIN_KEY, []])
 
     def test_grains_append(self):
         '''

tests/integration/modules/test_state.py

@@ -1732,6 +1732,39 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin):
         self.assertEqual(state_run[state_id]['comment'], 'Failure!')
         self.assertFalse(state_run[state_id]['result'])
 
+    def test_issue_46762_prereqs_on_a_state_with_unfulfilled_requirements(self):
+        '''
+        This tests the case where state C requires state A, which fails.
+        State C is a pre-required state for State B.
+        Since state A fails, state C will not run because the requisite failed,
+        therefore state B will not run because state C failed to run.
+
+        See https://github.com/saltstack/salt/issues/46762 for
+        more information.
+        '''
+        state_run = self.run_function(
+            'state.sls',
+            mods='issue-46762'
+        )
+
+        state_id = 'test_|-a_|-a_|-fail_without_changes'
+        self.assertIn(state_id, state_run)
+        self.assertEqual(state_run[state_id]['comment'],
+                         'Failure!')
+        self.assertFalse(state_run[state_id]['result'])
+
+        state_id = 'test_|-b_|-b_|-nop'
+        self.assertIn(state_id, state_run)
+        self.assertEqual(state_run[state_id]['comment'],
+                         'One or more requisite failed: issue-46762.c')
+        self.assertFalse(state_run[state_id]['result'])
+
+        state_id = 'test_|-c_|-c_|-nop'
+        self.assertIn(state_id, state_run)
+        self.assertEqual(state_run[state_id]['comment'],
+                         'One or more requisite failed: issue-46762.a')
+        self.assertFalse(state_run[state_id]['result'])
+
     def test_state_nonbase_environment(self):
         '''
         test state.sls with saltenv using a nonbase environment

tests/integration/runners/test_state.py

@@ -240,6 +240,19 @@ class StateRunnerTest(ShellCase):
 
         server_thread.join()
 
+    def test_orchestrate_subset(self):
+        '''
+        test orchestration state using subset
+        '''
+        ret = self.run_run('state.orchestrate orch.subset')
+
+        def count(thing, listobj):
+            return sum([obj.strip() == thing for obj in listobj])
+
+        self.assertEqual(count('ID: test subset', ret), 1)
+        self.assertEqual(count('Succeeded: 1', ret), 1)
+        self.assertEqual(count('Failed:    0', ret), 1)
+
 
 @skipIf(salt.utils.platform.is_windows(), '*NIX-only test')
 class OrchEventTest(ShellCase):

tests/unit/acl/test_client.py

@@ -19,9 +19,14 @@ class ClientACLTestCase(TestCase):
             'users': ['joker', 'penguin', '*bad_*', 'blocked_.*', '^Homer$'],
             'modules': ['cmd.run', 'test.fib', 'rm-rf.*'],
         }
+        self.whitelist = {
+            'users': ['testuser', 'saltuser'],
+            'modules': ['test.ping', 'grains.items'],
+        }
 
     def tearDown(self):
         del self.blacklist
+        del self.whitelist
 
     def test_user_is_blacklisted(self):
         '''
@@ -63,3 +68,25 @@ class ClientACLTestCase(TestCase):
 
         self.assertTrue(client_acl.cmd_is_blacklisted(['cmd.run', 'state.sls']))
         self.assertFalse(client_acl.cmd_is_blacklisted(['state.highstate', 'state.sls']))
+
+    def test_user_is_whitelisted(self):
+        '''
+        test user_is_whitelisted
+        '''
+        client_acl = acl.PublisherACL(self.whitelist)
+
+        self.assertTrue(client_acl.user_is_whitelisted('testuser'))
+        self.assertTrue(client_acl.user_is_whitelisted('saltuser'))
+        self.assertFalse(client_acl.user_is_whitelisted('three'))
+        self.assertFalse(client_acl.user_is_whitelisted('hans'))
+
+    def test_cmd_is_whitelisted(self):
+        '''
+        test cmd_is_whitelisted
+        '''
+        client_acl = acl.PublisherACL(self.whitelist)
+
+        self.assertTrue(client_acl.cmd_is_whitelisted('test.ping'))
+        self.assertTrue(client_acl.cmd_is_whitelisted('grains.items'))
+        self.assertFalse(client_acl.cmd_is_whitelisted('cmd.run'))
+        self.assertFalse(client_acl.cmd_is_whitelisted('test.version'))