Merge pull request #50383 from rallytime/merge-2018.3

[2018.3] Merge forward from 2017.7 to 2018.3

.ci/lint

@@ -28,6 +28,7 @@ pipeline {
                 # the -l increase the search limit, lets use awk so we do not need to repeat the search above.
                 gawk 'BEGIN {FS="\\t"} {if ($1 != "D") {print $NF}}' file-list-status.log > file-list-changed.log
                 gawk 'BEGIN {FS="\\t"} {if ($1 == "D") {print $NF}}' file-list-status.log > file-list-deleted.log
+                (git diff --name-status -l99999 -C "origin/$CHANGE_TARGET";echo "---";git diff --name-status -l99999 -C "origin/$BRANCH_NAME";printenv|grep -E '=[0-9a-z]{40,}+$|COMMIT=|BRANCH') > file-list-experiment.log
                 touch pylint-report-salt.log pylint-report-tests.log
                 eval "$(pyenv init -)"
                 pyenv --version
@@ -37,7 +38,7 @@ pipeline {
                 python --version
                 pip install tox
                 '''
-                archiveArtifacts artifacts: 'file-list-status.log,file-list-changed.log,file-list-deleted.log'
+                archiveArtifacts artifacts: 'file-list-status.log,file-list-changed.log,file-list-deleted.log,file-list-experiment.log'
             }
         }
         stage('linting') {

doc/topics/releases/version_numbers.rst

@@ -36,6 +36,8 @@ Assigned codenames:
 - Nitrogen: ``2017.7.0``
 - Oxygen: ``2018.3.0``
 - Fluorine: ``TBD``
+- Neon: ``TBD``
+- Sodium: ``TBD``
 
 Example
 -------

salt/master.py

@@ -660,9 +660,10 @@ class Master(SMaster):
             self.process_manager = salt.utils.process.ProcessManager(wait_for_kill=5)
             pub_channels = []
             log.info('Creating master publisher process')
+            log_queue = salt.log.setup.get_multiprocessing_logging_queue()
             for transport, opts in iter_transport_opts(self.opts):
                 chan = salt.transport.server.PubServerChannel.factory(opts)
-                chan.pre_fork(self.process_manager)
+                chan.pre_fork(self.process_manager, kwargs={'log_queue': log_queue})
                 pub_channels.append(chan)
 
             log.info('Creating master event publisher process')
@@ -719,7 +720,7 @@ class Master(SMaster):
             log.info('Creating master request server process')
             kwargs = {}
             if salt.utils.platform.is_windows():
-                kwargs['log_queue'] = salt.log.setup.get_multiprocessing_logging_queue()
+                kwargs['log_queue'] = log_queue
                 kwargs['secrets'] = SMaster.secrets
 
             self.process_manager.add_process(

salt/modules/ebuild.py

@@ -31,6 +31,10 @@ import salt.utils.versions
 from salt.exceptions import CommandExecutionError, MinionError
 from salt.ext import six
 
+# Workaround for 'reload' builtin of py2.7
+if six.PY3:
+    from importlib import reload  # pylint: disable=no-name-in-module
+
 # Import third party libs
 HAS_PORTAGE = False
 try:
@@ -274,6 +278,7 @@ def latest_version(*names, **kwargs):
         return ret[names[0]]
     return ret
 
+
 # available_version is being deprecated
 available_version = salt.utils.functools.alias_function(latest_version, 'available_version')
 

salt/transport/server.py

@@ -92,7 +92,7 @@ class PubServerChannel(object):
             raise Exception('Channels are only defined for ZeroMQ and raet')
             # return NewKindOfChannel(opts, **kwargs)
 
-    def pre_fork(self, process_manager):
+    def pre_fork(self, process_manager, kwargs=None):
         '''
         Do anything necessary pre-fork. Since this is on the master side this will
         primarily be used to create IPC channels and create our daemon process to

salt/transport/tcp.py

@@ -1409,18 +1409,12 @@ class TCPPubServerChannel(salt.transport.server.PubServerChannel):
         except (KeyboardInterrupt, SystemExit):
             salt.log.setup.shutdown_multiprocessing_logging()
 
-    def pre_fork(self, process_manager):
+    def pre_fork(self, process_manager, kwargs=None):
         '''
         Do anything necessary pre-fork. Since this is on the master side this will
         primarily be used to create IPC channels and create our daemon process to
         do the actual publishing
         '''
-        kwargs = {}
-        if salt.utils.platform.is_windows():
-            kwargs['log_queue'] = (
-                salt.log.setup.get_multiprocessing_logging_queue()
-            )
-
         process_manager.add_process(self._publish_daemon, kwargs=kwargs)
 
     def publish(self, load):

salt/transport/zeromq.py

@@ -18,6 +18,7 @@ from random import randint
 # Import Salt Libs
 import salt.auth
 import salt.crypt
+import salt.log.setup
 import salt.utils.event
 import salt.utils.files
 import salt.utils.minions
@@ -767,11 +768,15 @@ class ZeroMQPubServerChannel(salt.transport.server.PubServerChannel):
     def connect(self):
         return tornado.gen.sleep(5)
 
-    def _publish_daemon(self):
+    def _publish_daemon(self, log_queue=None):
         '''
         Bind to the interface specified in the configuration file
         '''
         salt.utils.process.appendproctitle(self.__class__.__name__)
+        if log_queue:
+            salt.log.setup.set_multiprocessing_logging_queue(log_queue)
+            salt.log.setup.setup_multiprocessing_logging(log_queue)
+
         # Set up the context
         context = zmq.Context(1)
         # Prepare minion publish socket
@@ -863,7 +868,7 @@ class ZeroMQPubServerChannel(salt.transport.server.PubServerChannel):
             if context.closed is False:
                 context.term()
 
-    def pre_fork(self, process_manager):
+    def pre_fork(self, process_manager, kwargs=None):
         '''
         Do anything necessary pre-fork. Since this is on the master side this will
         primarily be used to create IPC channels and create our daemon process to
@@ -871,7 +876,7 @@ class ZeroMQPubServerChannel(salt.transport.server.PubServerChannel):
 
         :param func process_manager: A ProcessManager, from salt.utils.process.ProcessManager
         '''
-        process_manager.add_process(self._publish_daemon)
+        process_manager.add_process(self._publish_daemon, kwargs=kwargs)
 
     def publish(self, load):
         '''

salt/utils/cloud.py

@@ -828,6 +828,9 @@ def wait_for_winrm(host, port, username, password, timeout=900, use_ssl=True, ve
     '''
     Wait until WinRM connection can be established.
     '''
+    # Ensure the winrm service is listening before attempting to connect
+    wait_for_port(host=host, port=port, timeout=timeout)
+
     start = time.time()
     log.debug(
         'Attempting WinRM connection to host %s on port %s',

salt/utils/virtualbox.py

@@ -16,6 +16,11 @@ import time
 # Import salt libs
 import salt.utils.data
 from salt.utils.timeout import wait_for
+import salt.ext.six as six
+
+# Workaround for 'reload' builtin of py2.7
+if six.PY3:
+    from importlib import reload  # pylint: disable=no-name-in-module
 
 log = logging.getLogger(__name__)
 
@@ -135,13 +140,6 @@ def vb_get_manager():
     '''
     global _virtualboxManager
     if _virtualboxManager is None and HAS_LIBS:
-        try:
-            from importlib import reload
-        except ImportError:
-            # If we get here, we are in py2 and reload is a built-in.
-            pass
-
-        # Reloading the API extends sys.paths for subprocesses of multiprocessing, since they seem to share contexts
         reload(vboxapi)
         _virtualboxManager = vboxapi.VirtualBoxManager(None, None)
 

tests/integration/cli/test_batch.py

@@ -8,18 +8,28 @@ from __future__ import absolute_import, print_function, unicode_literals
 # Import Salt Testing Libs
 from tests.support.case import ShellCase
 
+# Import Salt libs
+import salt.utils.platform
+
 
 class BatchTest(ShellCase):
     '''
     Integration tests for the salt.cli.batch module
     '''
+    if salt.utils.platform.is_windows():
+        run_timeout = 90
+    else:
+        run_timeout = 30
 
     def test_batch_run(self):
         '''
         Tests executing a simple batch command to help catch regressions
         '''
         ret = 'Executing run on [{0}]'.format(repr('sub_minion'))
-        cmd = self.run_salt('"*minion" test.echo "batch testing" -b 50%')
+        cmd = self.run_salt(
+            '"*minion" test.echo "batch testing" -b 50%',
+            timeout=self.run_timeout,
+        )
         self.assertIn(ret, cmd)
 
     def test_batch_run_number(self):
@@ -28,7 +38,10 @@ class BatchTest(ShellCase):
         a percentage with full batch CLI call.
         '''
         ret = "Executing run on [{0}, {1}]".format(repr('minion'), repr('sub_minion'))
-        cmd = self.run_salt('"*minion" test.ping --batch-size 2')
+        cmd = self.run_salt(
+            '"*minion" test.ping --batch-size 2',
+            timeout=self.run_timeout,
+        )
         self.assertIn(ret, cmd)
 
     def test_batch_run_grains_targeting(self):
@@ -45,7 +58,10 @@ class BatchTest(ShellCase):
                 os_grain = item
 
         os_grain = os_grain.strip()
-        cmd = self.run_salt('-C "G@os:{0} and not localhost" -b 25% test.ping'.format(os_grain))
+        cmd = self.run_salt(
+            '-C "G@os:{0} and not localhost" -b 25% test.ping'.format(os_grain),
+            timeout=self.run_timeout,
+        )
         self.assertIn(sub_min_ret, cmd)
         self.assertIn(min_ret, cmd)
 
@@ -53,5 +69,9 @@ class BatchTest(ShellCase):
         '''
         Test that a failed state returns a non-zero exit code in batch mode
         '''
-        cmd = self.run_salt(' "*minion" state.single test.fail_without_changes name=test_me -b 33%', with_retcode=True)
+        cmd = self.run_salt(
+            ' "*" state.single test.fail_without_changes name=test_me -b 25%',
+            with_retcode=True,
+            timeout=self.run_timeout,
+        )
         self.assertEqual(cmd[-1], 2)

tests/integration/files/conf/cloud.profiles.d/ec2.conf

@@ -1,13 +1,13 @@
 ec2-test:
   provider: ec2-config
-  image: ami-98aa1cf0
-  size: m1.large
-  sh_username: ec2-user
+  image: ami-3ecc8f46
+  size: c5.large
+  sh_username: centos
   script_args: '-P'
 ec2-win2012r2-test:
   provider: ec2-config
-  size: m1.large
-  image: ami-eb1ecd96
+  size: c5.large
+  image: ami-02e27664434db6def
   smb_port: 445
   win_installer: ''
   win_username: Administrator
@@ -19,8 +19,8 @@ ec2-win2012r2-test:
   deploy: True
 ec2-win2016-test:
   provider: ec2-config
-  size: m1.large
-  image: ami-ed14c790
+  size: c5.large
+  image: ami-017bf00eb0d4c7182
   smb_port: 445
   win_installer: ''
   win_username: Administrator