Merge branch '2019.2' into merge-2018.3

Gemfile

@@ -2,6 +2,7 @@
 
 source 'https://rubygems.org'
 
+# Point this back at the test-kitchen package after 1.23.3 is relased
 gem 'test-kitchen', '~>1.23.3'
 gem 'kitchen-salt', '~>0.4.1'
 gem 'kitchen-sync'

pkg/osx/distribution.xml.dist

@@ -12,9 +12,10 @@
              hostArchitectures="@CPUARCH@" />
     <domains enable_localSystem="true" />
     <!-- Define background image -->
-    <background file="saltstack.png"
+    <background file="logo.png"
                 mime-type="image/png"
-                scaling="proportional" />
+                scaling="proportional"
+                alignment="bottomleft" />
     <!-- Define documents displayed at various steps -->
     <welcome file="welcome@PY2@.rtf"
              mime-type="text/rtf" />

pkg/windows/installer/Salt-Minion-Setup.nsi

@@ -72,6 +72,7 @@ ${StrStrAdv}
 !define MUI_WELCOMEFINISHPAGE_BITMAP "panel.bmp"
 !define MUI_UNWELCOMEFINISHPAGE_BITMAP "panel.bmp"
 
+
 # This entire if block can be removed for the Sodium release... including the !define MUI_WELCOMEPAGE_TEXT
 # NSIS will just use the default like it does for Python 3, which should be the same test
 !if "${PYTHON_VERSION}" == "2"

salt/_compat.py

@@ -144,6 +144,12 @@ class IPv6AddressScoped(ipaddress.IPv6Address):
 
         :param address:
         '''
+        # pylint: disable-all
+        if not hasattr(self, '_is_packed_binary'):
+            # This method (below) won't be around for some Python 3 versions
+            # and we need check this differently anyway
+            self._is_packed_binary = lambda p: isinstance(p, bytes)
+        # pylint: enable-all
         if isinstance(address, string_types) and '%' in address:
             buff = address.split('%')
             if len(buff) != 2:

salt/scripts.py

@@ -20,6 +20,7 @@ from random import randint
 # Import salt libs
 from salt.exceptions import SaltSystemExit, SaltClientError, SaltReqTimeoutError
 import salt.defaults.exitcodes  # pylint: disable=unused-import
+import salt.ext.six as six
 
 log = logging.getLogger(__name__)
 
@@ -93,6 +94,16 @@ def salt_master():
     Start the salt master.
     '''
     import salt.cli.daemons
+# REMOVEME after Python 2.7 support is dropped (also the six import)
+    if six.PY2:
+        from salt.utils.versions import warn_until
+        # Message borrowed from pip's deprecation warning
+        warn_until('Sodium',
+                   'Python 2.7 will reach the end of its life on January 1st,'
+                   ' 2020. Please upgrade your Python as Python 2.7 won\'t be'
+                   ' maintained after that date.  Salt will drop support for'
+                   ' Python 2.7 in the Sodium release or later.')
+# END REMOVEME
     master = salt.cli.daemons.Master()
     master.start()
 
@@ -179,6 +190,16 @@ def salt_minion():
         minion = salt.cli.daemons.Minion()
         minion.start()
         return
+# REMOVEME after Python 2.7 support is dropped (also the six import)
+    elif six.PY2:
+        from salt.utils.versions import warn_until
+        # Message borrowed from pip's deprecation warning
+        warn_until('Sodium',
+                   'Python 2.7 will reach the end of its life on January 1st,'
+                   ' 2020. Please upgrade your Python as Python 2.7 won\'t be'
+                   ' maintained after that date.  Salt will drop support for'
+                   ' Python 2.7 in the Sodium release or later.')
+# END REMOVEME
 
     if '--disable-keepalive' in sys.argv:
         sys.argv.remove('--disable-keepalive')

salt/transport/ipc.py

@@ -219,7 +219,12 @@ class IPCServer(object):
             self.sock.close()
 
     def __del__(self):
-        self.close()
+        try:
+            self.close()
+        except TypeError:
+            # This is raised when Python's GC has collected objects which
+            # would be needed when calling self.close()
+            pass
 
 
 class IPCClient(object):
@@ -360,16 +365,21 @@ class IPCClient(object):
                 yield tornado.gen.sleep(1)
 
     def __del__(self):
-        with self._refcount_lock:
-            # Make sure we actually close no matter if something
-            # went wrong with our ref counting
-            self._refcount = 1
         try:
-            self.close()
-        except socket.error as exc:
-            if exc.errno != errno.EBADF:
-                # If its not a bad file descriptor error, raise
-                raise
+            with self._refcount_lock:
+                # Make sure we actually close no matter if something
+                # went wrong with our ref counting
+                self._refcount = 1
+            try:
+                self.close()
+            except socket.error as exc:
+                if exc.errno != errno.EBADF:
+                    # If its not a bad file descriptor error, raise
+                    raise
+        except TypeError:
+            # This is raised when Python's GC has collected objects which
+            # would be needed when calling self.close()
+            pass
 
     def close(self):
         '''
@@ -610,7 +620,12 @@ class IPCMessagePublisher(object):
             self.sock.close()
 
     def __del__(self):
-        self.close()
+        try:
+            self.close()
+        except TypeError:
+            # This is raised when Python's GC has collected objects which
+            # would be needed when calling self.close()
+            pass
 
 
 class IPCMessageSubscriber(IPCClient):

tests/integration/__init__.py

@@ -183,7 +183,7 @@ class TestDaemon(object):
     '''
     Set up the master and minion daemons, and run related cases
     '''
-    MINIONS_CONNECT_TIMEOUT = MINIONS_SYNC_TIMEOUT = 120
+    MINIONS_CONNECT_TIMEOUT = MINIONS_SYNC_TIMEOUT = 300
 
     def __init__(self, parser):
         self.parser = parser
@@ -219,6 +219,8 @@ class TestDaemon(object):
         if getattr(self.parser.options, 'ssh', False):
             self.prep_ssh()
 
+        self.wait_for_minions(time.time(), self.MINIONS_CONNECT_TIMEOUT)
+
         if self.parser.options.sysinfo:
             try:
                 print_header(
@@ -1185,84 +1187,6 @@ class TestDaemon(object):
             k for (k, v) in six.iteritems(running) if v and v[0]['jid'] == jid
         ]
 
-    def wait_for_minion_connections(self, targets, timeout):
-        salt.utils.process.appendproctitle('WaitForMinionConnections')
-        sys.stdout.write(
-            ' {LIGHT_BLUE}*{ENDC} Waiting at most {0} for minions({1}) to '
-            'connect back\n'.format(
-                (timeout > 60 and
-                 timedelta(seconds=timeout) or
-                 '{0} secs'.format(timeout)),
-                ', '.join(targets),
-                **self.colors
-            )
-        )
-        sys.stdout.flush()
-        expected_connections = set(targets)
-        now = datetime.now()
-        expire = now + timedelta(seconds=timeout)
-        while now <= expire:
-            sys.stdout.write(
-                '\r{0}\r'.format(
-                    ' ' * getattr(self.parser.options, 'output_columns', PNUM)
-                )
-            )
-            sys.stdout.write(
-                ' * {LIGHT_YELLOW}[Quit in {0}]{ENDC} Waiting for {1}'.format(
-                    '{0}'.format(expire - now).rsplit('.', 1)[0],
-                    ', '.join(expected_connections),
-                    **self.colors
-                )
-            )
-            sys.stdout.flush()
-
-            try:
-                responses = self.client.cmd(
-                    list(expected_connections), 'test.ping', tgt_type='list',
-                )
-            # we'll get this exception if the master process hasn't finished starting yet
-            except SaltClientError:
-                time.sleep(0.1)
-                now = datetime.now()
-                continue
-            for target in responses:
-                if target not in expected_connections:
-                    # Someone(minion) else "listening"?
-                    continue
-                expected_connections.remove(target)
-                sys.stdout.write(
-                    '\r{0}\r'.format(
-                        ' ' * getattr(self.parser.options, 'output_columns',
-                                      PNUM)
-                    )
-                )
-                sys.stdout.write(
-                    '   {LIGHT_GREEN}*{ENDC} {0} connected.\n'.format(
-                        target, **self.colors
-                    )
-                )
-                sys.stdout.flush()
-
-            if not expected_connections:
-                return
-
-            time.sleep(1)
-            now = datetime.now()
-        else:  # pylint: disable=W0120
-            print(
-                '\n {LIGHT_RED}*{ENDC} WARNING: Minions failed to connect '
-                'back. Tests requiring them WILL fail'.format(**self.colors)
-            )
-            try:
-                print_header(
-                    '=', sep='=', inline=True,
-                    width=getattr(self.parser.options, 'output_columns', PNUM)
-
-                )
-            except TypeError:
-                print_header('=', sep='=', inline=True)
-            raise SystemExit()
-
     def sync_minion_modules_(self, modules_kind, targets, timeout=None):
         if not timeout:
             timeout = 120
@@ -1339,3 +1263,20 @@ class TestDaemon(object):
     def sync_minion_grains(self, targets, timeout=None):
         salt.utils.process.appendproctitle('SyncMinionGrains')
         self.sync_minion_modules_('grains', targets, timeout=timeout)
+
+    def wait_for_minions(self, start, timeout, sleep=5):
+        '''
+        Ensure all minions and masters (including sub-masters) are connected.
+        '''
+        while True:
+            try:
+                ret = self.client.run_job('*', 'test.ping')
+            except salt.exceptions.SaltClientError:
+                ret = None
+            if ret and 'minions' not in ret:
+                continue
+            if ret and sorted(ret['minions']) == ['minion', 'sub_minion']:
+                break
+            if time.time() - start >= timeout:
+                raise RuntimeError("Ping Minions Failed")
+            time.sleep(sleep)

tests/integration/states/test_file.py

@@ -148,7 +148,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin):
         remove files created in previous tests
         '''
         user = 'salt'
-        if user in str(self.run_function('user.list_users', [user])):
+        if user in str(self.run_function('user.list_users')):
             self.run_function('user.delete', [user])
 
         for path in (FILEPILLAR, FILEPILLARDEF, FILEPILLARGIT):

tests/support/case.py

@@ -276,9 +276,9 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
             if 'env' not in popen_kwargs:
                 popen_kwargs['env'] = os.environ.copy()
                 if sys.version_info[0] < 3:
-                    popen_kwargs['env'][b'PYTHONPATH'] = os.getcwd().encode()
+                    popen_kwargs['env'][b'PYTHONPATH'] = CODE_DIR.encode()
                 else:
-                    popen_kwargs['env']['PYTHONPATH'] = os.getcwd()
+                    popen_kwargs['env']['PYTHONPATH'] = CODE_DIR
         else:
             cmd = 'PYTHONPATH='
             python_path = os.environ.get('PYTHONPATH', None)

tests/unit/grains/test_core.py

@@ -601,7 +601,7 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin):
         self.assertIn(returned_grains['windowsdomaintype'], valid_types)
         valid_releases = ['Vista', '7', '8', '8.1', '10', '2008Server',
                           '2008ServerR2', '2012Server', '2012ServerR2',
-                          '2016Server']
+                          '2016Server', '2019Server']
         self.assertIn(returned_grains['osrelease'], valid_releases)
 
     @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux')