Merge pull request #45609 from rallytime/merge-2017.7

[2017.7] Merge forward from 2017.7.3 to 2017.7

salt/client/ssh/ssh_py_shim.py

@@ -157,7 +157,10 @@ def unpack_thin(thin_path):
     tfile.extractall(path=OPTIONS.saltdir)
     tfile.close()
     os.umask(old_umask)
-    os.unlink(thin_path)
+    try:
+        os.unlink(thin_path)
+    except OSError:
+        pass
 
 
 def need_ext():

salt/engines/__init__.py

@@ -127,4 +127,7 @@ class Engine(SignalHandlingMultiprocessingProcess):
         try:
             self.engine[self.fun](**kwargs)
         except Exception as exc:
-            log.critical('Engine {0} could not be started! Error: {1}'.format(self.engine, exc))
+            log.critical(
+                'Engine \'%s\' could not be started!',
+                self.fun.split('.')[0], exc_info=True
+            )

salt/modules/pip.py

@@ -82,7 +82,7 @@ import re
 import shutil
 import logging
 import sys
-from pkg_resources import parse_version
+import pkg_resources
 
 # Import salt libs
 import salt.utils
@@ -1287,7 +1287,7 @@ def list_all_versions(pkg,
         match = re.search(r'\s*Could not find a version.* \(from versions: (.*)\)', line)
         if match:
             versions = [v for v in match.group(1).split(', ') if v and excludes.match(v)]
-            versions.sort(key=parse_version)
+            versions.sort(key=pkg_resources.parse_version)
             break
     if not versions:
         return None

salt/states/pip_state.py

@@ -23,7 +23,7 @@ requisite to a pkg.installed state for the package which provides pip
 from __future__ import absolute_import
 import re
 import logging
-from pkg_resources import parse_version
+import pkg_resources
 
 # Import salt libs
 import salt.utils
@@ -266,11 +266,11 @@ def _pep440_version_cmp(pkg1, pkg2, ignore_epoch=False):
     pkg2 = normalize(pkg2)
 
     try:
-        if parse_version(pkg1) < parse_version(pkg2):
+        if pkg_resources.parse_version(pkg1) < pkg_resources.parse_version(pkg2):
             return -1
-        if parse_version(pkg1) == parse_version(pkg2):
+        if pkg_resources.parse_version(pkg1) == pkg_resources.parse_version(pkg2):
             return 0
-        if parse_version(pkg1) > parse_version(pkg2):
+        if pkg_resources.parse_version(pkg1) > pkg_resources.parse_version(pkg2):
             return 1
     except Exception as exc:
         logger.exception(exc)

salt/transport/zeromq.py

@@ -16,6 +16,8 @@ import weakref
 from random import randint
 
 # Import Salt Libs
+from salt.ext import six
+from salt.ext.six.moves import map
 import salt.auth
 import salt.crypt
 import salt.utils
@@ -40,13 +42,15 @@ try:
 except ImportError:
     HAS_ZMQ_MONITOR = False
 
+LIBZMQ_VERSION = tuple(map(int, zmq.zmq_version().split('.')))
+PYZMQ_VERSION = tuple(map(int, zmq.pyzmq_version().split('.')))
+
 # Import Tornado Libs
 import tornado
 import tornado.gen
 import tornado.concurrent
 
 # Import third party libs
-import salt.ext.six as six
 try:
     from Cryptodome.Cipher import PKCS1_OAEP
 except ImportError:
@@ -359,7 +363,12 @@ class AsyncZeroMQPubChannel(salt.transport.mixins.auth.AESPubClientMixin, salt.t
             self._monitor.stop()
             self._monitor = None
         if hasattr(self, '_stream'):
-            self._stream.close(0)
+            if PYZMQ_VERSION < (14, 3, 0):
+                # stream.close() doesn't work properly on pyzmq < 14.3.0
+                self._stream.io_loop.remove_handler(self._stream.socket)
+                self._stream.socket.close(0)
+            else:
+                self._stream.close(0)
         elif hasattr(self, '_socket'):
             self._socket.close(0)
         if hasattr(self, 'context') and self.context.closed is False:
@@ -911,8 +920,17 @@ class AsyncReqMessageClient(object):
     # TODO: timeout all in-flight sessions, or error
     def destroy(self):
         if hasattr(self, 'stream') and self.stream is not None:
-            self.stream.close()
-            self.socket = None
+            if PYZMQ_VERSION < (14, 3, 0):
+                # stream.close() doesn't work properly on pyzmq < 14.3.0
+                if self.stream.socket:
+                    self.stream.socket.close()
+                self.stream.io_loop.remove_handler(self.stream.socket)
+                # set this to None, more hacks for messed up pyzmq
+                self.stream.socket = None
+                self.socket.close()
+            else:
+                self.stream.close()
+                self.socket = None
             self.stream = None
         if self.context.closed is False:
             self.context.term()

salt/utils/event.py

@@ -91,8 +91,8 @@ SUB_EVENT = set([
     'state.sls',
 ])
 
-TAGEND = '\n\n'  # long tag delimiter
-TAGPARTER = '/'  # name spaced tag delimiter
+TAGEND = str('\n\n')  # long tag delimiter
+TAGPARTER = str('/')  # name spaced tag delimiter
 SALT = 'salt'  # base prefix for all salt/ events
 # dict map of namespaced base tag prefixes for salt events
 TAGS = {
@@ -725,14 +725,11 @@ class SaltEvent(object):
             is_msgpacked=True,
             use_bin_type=six.PY3
         )
-        log.debug('Sending event: tag = {0}; data = {1}'.format(tag, data))
-        if six.PY2:
-            event = '{0}{1}{2}'.format(tag, tagend, serialized_data)
-        else:
-            event = b''.join([
-                salt.utils.to_bytes(tag),
-                salt.utils.to_bytes(tagend),
-                serialized_data])
+        log.debug('Sending event: tag = %s; data = %s', tag, data)
+        event = b''.join([
+            salt.utils.to_bytes(tag),
+            salt.utils.to_bytes(tagend),
+            serialized_data])
         msg = salt.utils.to_bytes(event, 'utf-8')
         if self._run_io_loop_sync:
             with salt.utils.async.current_ioloop(self.io_loop):

salt/utils/thin.py

@@ -534,6 +534,7 @@ def gen_min(cachedir, extra_mods='', overwrite=False, so_mods='',
         'salt/modules/test.py',
         'salt/modules/selinux.py',
         'salt/modules/cmdmod.py',
+        'salt/modules/saltutil.py',
         'salt/minion.py',
         'salt/pillar',
         'salt/pillar/__init__.py',

tests/integration/files/file/base/running.sls

@@ -1,4 +1,4 @@
 sleep_running:
   module.run:
     - name: test.sleep
-    - length: 10
+    - length: 20

tests/integration/minion/test_blackout.py

@@ -12,7 +12,7 @@ import textwrap
 # Import Salt Testing libs
 from tests.support.case import ModuleCase
 from tests.support.paths import PILLAR_DIR
-from tests.support.helpers import destructiveTest
+from tests.support.helpers import destructiveTest, flaky
 
 # Import Salt libs
 import salt.utils
@@ -33,7 +33,7 @@ class MinionBlackoutTestCase(ModuleCase):
         with salt.utils.fopen(BLACKOUT_PILLAR, 'w') as wfh:
             wfh.write(blackout_data)
         self.run_function('saltutil.refresh_pillar')
-        sleep(5)  # wait for minion to enter blackout mode
+        sleep(10)  # wait for minion to enter blackout mode
 
     def end_blackout(self):
         '''
@@ -44,8 +44,9 @@ class MinionBlackoutTestCase(ModuleCase):
                 minion_blackout: False
                 '''))
         self.run_function('saltutil.refresh_pillar')
-        sleep(5)  # wait for minion to exit blackout mode
+        sleep(10)  # wait for minion to exit blackout mode
 
+    @flaky
     def test_blackout(self):
         '''
         Test that basic minion blackout functionality works
@@ -60,6 +61,7 @@ class MinionBlackoutTestCase(ModuleCase):
         ret = self.run_function('test.ping')
         self.assertEqual(ret, True)
 
+    @flaky
     def test_blackout_whitelist(self):
         '''
         Test that minion blackout whitelist works
@@ -81,6 +83,7 @@ class MinionBlackoutTestCase(ModuleCase):
         finally:
             self.end_blackout()
 
+    @flaky
     def test_blackout_nonwhitelist(self):
         '''
         Test that minion refuses to run non-whitelisted functions during

tests/integration/modules/test_service.py

@@ -23,7 +23,7 @@ class ServiceModuleTest(ModuleCase):
         if os_family == 'RedHat':
             self.service_name = 'crond'
         elif os_family == 'Arch':
-            self.service_name = 'systemd-journald'
+            self.service_name = 'sshd'
             cmd_name = 'systemctl'
         elif os_family == 'MacOS':
             self.service_name = 'org.ntp.ntpd'

tests/integration/spm/test_man_spm.py

@@ -4,19 +4,41 @@ Tests man spm
 '''
 # Import python libs
 from __future__ import absolute_import
+import os
+import shutil
+import sys
+import tempfile
 
 # Import Salt Testing libs
 from tests.support.case import ModuleCase
+from tests.support.helpers import destructiveTest
+from tests.support.paths import CODE_DIR
 
 
+@destructiveTest
 class SPMManTest(ModuleCase):
     '''
     Validate man spm
     '''
+
+    def setUp(self):
+        self.tmpdir = tempfile.mktemp()
+        os.mkdir(self.tmpdir)
+        self.run_function('cmd.run', ['{0} {1} install --root={2}'.format(
+            sys.executable,
+            os.path.join(CODE_DIR, 'setup.py'),
+            self.tmpdir
+        )])
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir)
+
     def test_man_spm(self):
         '''
         test man spm
         '''
-        cmd = self.run_function('cmd.run', ['man spm'])
+        manpath = self.run_function('cmd.run', ['find {0} -name spm.1'.format(self.tmpdir)])
+        self.assertIn('/man1/', manpath)
+        cmd = self.run_function('cmd.run', ['man {0}'.format(manpath)])
         self.assertIn('Salt Package Manager', cmd)
         self.assertIn('command for managing Salt packages', cmd)

tests/integration/ssh/test_state.py

@@ -4,6 +4,7 @@
 from __future__ import absolute_import
 import os
 import shutil
+import threading
 import time
 
 # Import Salt Testing Libs
@@ -13,6 +14,7 @@ from tests.support.unit import skipIf
 
 # Import Salt Libs
 from salt.ext import six
+from salt.ext.six.moves import range  # pylint: disable=redefined-builtin
 
 SSH_SLS = 'ssh_state_tests'
 SSH_SLS_FILE = '/tmp/test'
@@ -166,19 +168,34 @@ class SSHStateTest(SSHCase):
         '''
         test state.running with salt-ssh
         '''
-        start_sls = self.run_function('state.sls', ['running', '&'],
-                                      wipe=False)
-        time.sleep(8)
-        get_sls = self.run_function('state.running', wipe=False)
-        ret = 'The function "state.pkg" is running as'
-        self.assertIn(ret, ' '.join(get_sls))
+        def _run_in_background():
+            self.run_function('state.sls', ['running'], wipe=False)
+
+        bg_thread = threading.Thread(target=_run_in_background)
+        bg_thread.start()
+
+        expected = 'The function "state.pkg" is running as'
+        for _ in range(3):
+            time.sleep(5)
+            get_sls = self.run_function('state.running', wipe=False)
+            try:
+                self.assertIn(expected, ' '.join(get_sls))
+            except AssertionError:
+                pass
+            else:
+                # We found the expected return
+                break
+        else:
+            self.fail(
+                'Did not find \'{0}\' in state.running return'.format(expected)
+            )
 
         # make sure we wait until the earlier state is complete
         future = time.time() + 120
         while True:
             if time.time() > future:
                 break
-            if ret not in ' '.join(self.run_function('state.running', wipe=False)):
+            if expected not in ' '.join(self.run_function('state.running', wipe=False)):
                 break
 
     def tearDown(self):

tests/integration/states/test_service.py

@@ -31,7 +31,7 @@ class ServiceTest(ModuleCase, SaltReturnAssertsMixin):
         if os_family == 'RedHat':
             self.service_name = 'crond'
         elif os_family == 'Arch':
-            self.service_name = 'systemd-journald'
+            self.service_name = 'sshd'
             cmd_name = 'systemctl'
         elif os_family == 'MacOS':
             self.service_name = 'org.ntp.ntpd'

tests/unit/templates/test_jinja.py

@@ -892,7 +892,7 @@ class TestCustomExtensions(TestCase):
     def test_http_query(self):
         '''Test the `http_query` Jinja filter.'''
         for backend in ('requests', 'tornado', 'urllib2'):
-            rendered = render_jinja_tmpl("{{ 'http://www.google.com' | http_query(backend='" + backend + "') }}",
+            rendered = render_jinja_tmpl("{{ 'http://icanhazip.com' | http_query(backend='" + backend + "') }}",
                                          dict(opts=self.local_opts, saltenv='test', salt=self.local_salt))
             self.assertIsInstance(rendered, six.text_type, 'Failed with backend: {}'.format(backend))
             dict_reply = ast.literal_eval(rendered)