Merge pull request #48252 from rallytime/merge-2018.3

[2018.3] Merge forward from 2017.7 to 2018.3

doc/ref/configuration/minion.rst

@@ -1347,6 +1347,39 @@ The password used for HTTP proxy access.
 
     proxy_password: obolus
 
+Docker Configuration
+====================
+
+.. conf_minion:: docker.update_mine
+
+``docker.update_mine``
+----------------------
+
+.. versionadded:: 2017.7.8,2018.3.3
+.. versionchanged:: Fluorine
+    The default value is now ``False``
+
+Default: ``True``
+
+If enabled, when containers are added, removed, stopped, started, etc., the
+:ref:`mine <salt-mine>` will be updated with the results of :py:func:`docker.ps
+verbose=True all=True host=True <salt.modules.dockermod.ps>`. This mine data is
+used by :py:func:`mine.get_docker <salt.modules.mine.get_docker>`. Set this
+option to ``False`` to keep Salt from updating the mine with this information.
+
+.. note::
+    This option can also be set in Grains or Pillar data, with Grains
+    overriding Pillar and the minion config file overriding Grains.
+
+.. note::
+    Disabling this will of course keep :py:func:`mine.get_docker
+    <salt.modules.mine.get_docker>` from returning any information for a given
+    minion.
+
+.. code-block:: yaml
+
+    docker.update_mine: False
+
 .. conf_minion:: docker.compare_container_networks
 
 ``docker.compare_container_networks``
@@ -1381,6 +1414,7 @@ Specifies which keys are examined by
         - GlobalIPv6Address
         - IPv6Gateway
 
+
 Minion Execution Module Management
 ==================================
 

doc/topics/releases/2017.7.8.rst

@@ -16,3 +16,16 @@ New win_snmp behavior
 - :py:func:`win_snmp.set_community_names
   <salt.modules.win_snmp.set_community_names>` now raises an error when SNMP
   settings are being managed by GroupPolicy.
+
+Option Added to Disable Docker Mine Updates
+===========================================
+
+When a docker container is added, removed, started, stopped, etc., the results
+of a :py:func:`docker.ps verbose=True all=True host=True
+<salt.modules.dockermod.ps>` are sent to the :ref:`mine <salt-mine>`, to be
+used by :py:func:`mine.get_docker <salt.modules.mine.get_docker>`.
+
+A new config option (:conf_minion:`docker.update_mine`) has been added.  When
+set to ``False``, Salt will not send this information to the mine. This is
+useful in cases where sensitive information is stored in the container's
+environment.

salt/minion.py

@@ -1172,10 +1172,11 @@ class Minion(MinionBase):
         # I made the following 3 line oddity to preserve traceback.
         # Please read PR #23978 before changing, hopefully avoiding regressions.
         # Good luck, we're all counting on you.  Thanks.
-        future_exception = self._connect_master_future.exception()
-        if future_exception:
-            # This needs to be re-raised to preserve restart_on_error behavior.
-            raise six.reraise(*future_exception)
+        if self._connect_master_future.done():
+            future_exception = self._connect_master_future.exception()
+            if future_exception:
+                # This needs to be re-raised to preserve restart_on_error behavior.
+                raise six.reraise(*future_exception)
         if timeout and self._sync_connect_master_success is False:
             raise SaltDaemonNotRunning('Failed to connect to the salt-master')
 
@@ -1655,7 +1656,9 @@ class Minion(MinionBase):
                 )
                 ret['out'] = 'nested'
             except TypeError as exc:
-                msg = 'Passed invalid arguments to {0}: {1}\n{2}'.format(function_name, exc, func.__doc__)
+                msg = 'Passed invalid arguments to {0}: {1}\n{2}'.format(
+                    function_name, exc, func.__doc__ or ''
+                )
                 log.warning(msg, exc_info_on_loglevel=logging.DEBUG)
                 ret['return'] = msg
                 ret['out'] = 'nested'

salt/modules/apk.py

@@ -304,7 +304,7 @@ def install(name=None,
         # We don't support installing specific version for now
         # so transform the dict in list ignoring version provided
         pkgs = [
-            p.keys()[0] for p in pkgs
+            next(iter(p)) for p in pkgs
             if isinstance(p, dict)
         ]
         pkg_to_install.extend(pkgs)

salt/modules/dockermod.py

@@ -436,11 +436,20 @@ def _refresh_mine_cache(wrapped):
         refresh salt mine on exit.
         '''
         returned = wrapped(*args, **__utils__['args.clean_kwargs'](**kwargs))
-        __salt__['mine.send']('docker.ps', verbose=True, all=True, host=True)
+        if _check_update_mine():
+            __salt__['mine.send']('docker.ps', verbose=True, all=True, host=True)
         return returned
     return wrapper
 
 
+def _check_update_mine():
+    try:
+        ret = __context__['docker.update_mine']
+    except KeyError:
+        ret = __context__['docker.update_mine'] = __salt__['config.get']('docker.update_mine', default=True)
+    return ret
+
+
 # Helper functions
 def _change_state(name, action, expected, *args, **kwargs):
     '''

salt/modules/lxc.py

@@ -3504,7 +3504,9 @@ def bootstrap(name,
                 configdir = '/var/tmp/.c_{0}'.format(rstr)
 
                 cmd = 'install -m 0700 -d {0}'.format(configdir)
-                if run(name, cmd, python_shell=False):
+                if run_all(
+                    name, cmd, path=path, python_shell=False
+                )['retcode'] != 0:
                     log.error('tmpdir %s creation failed %s', configdir, cmd)
                     return False
 
@@ -3514,6 +3516,7 @@ def bootstrap(name,
                 copy_to(name, bs_, script, path=path)
                 result = run_all(name,
                                  'sh -c "chmod +x {0}"'.format(script),
+                                 path=path,
                                  python_shell=True)
 
                 copy_to(name, cfg_files['config'],
@@ -3539,6 +3542,7 @@ def bootstrap(name,
                 run_all(name,
                         'sh -c \'if [ -f "{0}" ];then rm -f "{0}";fi\''
                         ''.format(script),
+                        path=path,
                         ignore_retcode=True,
                         python_shell=True)
             else:

salt/modules/mine.py

@@ -374,10 +374,18 @@ def flush():
 
 def get_docker(interfaces=None, cidrs=None, with_container_id=False):
     '''
-    Get all mine data for 'docker.get_containers' and run an aggregation
-    routine. The "interfaces" parameter allows for specifying which network
-    interfaces to select ip addresses from. The "cidrs" parameter allows for
-    specifying a list of cidrs which the ip address must match.
+    .. versionchanged:: 2017.7.8,2018.3.3
+        When :conf_minion:`docker.update_mine` is set to ``False`` for a given
+        minion, no mine data will be populated for that minion, and thus none
+        will be returned for it.
+    .. versionchanged:: Fluorine
+        :conf_minion:`docker.update_mine` now defaults to ``False``
+
+    Get all mine data for :py:func:`docker.ps <salt.modules.dockermod.ps_>` and
+    run an aggregation routine. The ``interfaces`` parameter allows for
+    specifying the network interfaces from which to select IP addresses. The
+    ``cidrs`` parameter allows for specifying a list of subnets which the IP
+    address must match.
 
     with_container_id
         Boolean, to expose container_id in the list of results

salt/transport/ipc.py

@@ -751,9 +751,9 @@ class IPCMessageSubscriber(IPCClient):
             # This will prevent this message from showing up:
             # '[ERROR   ] Future exception was never retrieved:
             # StreamClosedError'
-            if self._read_sync_future is not None:
+            if self._read_sync_future is not None and self._read_sync_future.done():
                 self._read_sync_future.exception()
-            if self._read_stream_future is not None:
+            if self._read_stream_future is not None and self._read_stream_future.done():
                 self._read_stream_future.exception()
 
     def __del__(self):

salt/transport/tcp.py

@@ -909,10 +909,9 @@ class SaltMessageClient(object):
                     # This happens because the logic is always waiting to read
                     # the next message and the associated read future is marked
                     # 'StreamClosedError' when the stream is closed.
-                    self._read_until_future.exception()
-                    if (not self._stream_return_future.done() and
-                            self.io_loop != tornado.ioloop.IOLoop.current(
-                                instance=False)):
+                    if self._read_until_future.done():
+                        self._read_until_future.exception()
+                    elif self.io_loop != tornado.ioloop.IOLoop.current(instance=False):
                         self.io_loop.add_future(
                             self._stream_return_future,
                             lambda future: self.io_loop.stop()
@@ -1158,7 +1157,7 @@ class Subscriber(object):
         self._closing = True
         if not self.stream.closed():
             self.stream.close()
-            if self._read_until_future is not None:
+            if self._read_until_future is not None and self._read_until_future.done():
                 # This will prevent this message from showing up:
                 # '[ERROR   ] Future exception was never retrieved:
                 # StreamClosedError'

salt/utils/functools.py

@@ -56,6 +56,6 @@ def alias_function(fun, name, doc=None):
         orig_name = fun.__name__
         alias_msg = ('\nThis function is an alias of '
                      '``{0}``.\n'.format(orig_name))
-        alias_fun.__doc__ = alias_msg + fun.__doc__
+        alias_fun.__doc__ = alias_msg + (fun.__doc__ or '')
 
     return alias_fun

tests/support/win_installer.py

@@ -29,7 +29,7 @@ def iter_installers(content):
         x = m.groups()[0]
         if not x.startswith(PREFIX):
             continue
-        if x.endswith('zip'):
+        if x.endswith(('zip', 'sha256')):
             continue
         if installer:
             if x != installer + '.md5':

tests/unit/modules/test_dockermod.py

@@ -165,6 +165,7 @@ class DockerTestCase(TestCase, LoaderModuleMockMixin):
                 with patch.dict(docker_mod.__salt__,
                                 {'mine.send': mine_send,
                                  'container_resource.run': MagicMock(),
+                                 'config.get': MagicMock(return_value=True),
                                  'cp.cache_file': MagicMock(return_value=False)}):
                     with patch.dict(docker_mod.__utils__,
                                     {'docker.get_client_args': client_args_mock}):
@@ -173,6 +174,44 @@ class DockerTestCase(TestCase, LoaderModuleMockMixin):
                 mine_send.assert_called_with('docker.ps', verbose=True, all=True,
                                              host=True)
 
+    def test_update_mine(self):
+        '''
+        Test the docker.update_mine config option
+        '''
+        def config_get_disabled(val, default):
+            return {'base_url': docker_mod.NOTSET,
+                    'version': docker_mod.NOTSET,
+                    'docker.url': docker_mod.NOTSET,
+                    'docker.version': docker_mod.NOTSET,
+                    'docker.machine': docker_mod.NOTSET,
+                    'docker.update_mine': False}[val]
+
+        def config_get_enabled(val, default):
+            return {'base_url': docker_mod.NOTSET,
+                    'version': docker_mod.NOTSET,
+                    'docker.url': docker_mod.NOTSET,
+                    'docker.version': docker_mod.NOTSET,
+                    'docker.machine': docker_mod.NOTSET,
+                    'docker.update_mine': True}[val]
+
+        mine_mock = Mock()
+        dunder_salt = {
+            'config.get': MagicMock(side_effect=config_get_disabled),
+            'mine.send': mine_mock,
+        }
+        with patch.dict(docker_mod.__salt__, dunder_salt), \
+                patch.dict(docker_mod.__context__, {'docker.client': Mock()}), \
+                patch.object(docker_mod, 'state', MagicMock(return_value='stopped')):
+            docker_mod.stop('foo', timeout=1)
+            mine_mock.assert_not_called()
+
+        with patch.dict(docker_mod.__salt__, dunder_salt), \
+                patch.dict(docker_mod.__context__, {'docker.client': Mock()}), \
+                patch.object(docker_mod, 'state', MagicMock(return_value='stopped')):
+            dunder_salt['config.get'].side_effect = config_get_enabled
+            docker_mod.stop('foo', timeout=1)
+            self.assert_called_once(mine_mock)
+
     @skipIf(_docker_py_version() < (1, 5, 0),
             'docker module must be installed to run this test or is too old. >=1.5.0')
     def test_list_networks(self, *args):