Merge pull request #39577 from rallytime/merge-2016.11

[2016.11] Merge forward from 2016.3 to 2016.11

.mention-bot

@@ -0,0 +1,6 @@
+{
+ "skipTitle": "Merge forward",
+ "delayed": true,
+ "delayedUntil": "2h"
+}
+

.pylintrc

@@ -52,7 +52,8 @@ fileperms-default=0644
 fileperms-ignore-paths=tests/runtests.py,tests/jenkins*.py,tests/saltsh.py,tests/buildpackage.py
 
 # Py3 Modernize PyLint Plugin Settings
-modernize-nofix = libmodernize.fixes.fix_dict_six
+modernize-nofix = libmodernize.fixes.fix_dict_six,
+  saltpylint.py3modernize.fixes.fix_dict_salt_six
 
 # Minimum Python Version To Enforce
 minimum-python-version = 2.6

.testing.pylintrc

@@ -49,7 +49,8 @@ fileperms-default=0644
 fileperms-ignore-paths=tests/runtests.py,tests/jenkins*.py,tests/saltsh.py,tests/buildpackage.py
 
 # Py3 Modernize PyLint Plugin Settings
-modernize-nofix = libmodernize.fixes.fix_dict_six
+modernize-nofix = libmodernize.fixes.fix_dict_six,
+  saltpylint.py3modernize.fixes.fix_dict_salt_six
 
 # Minimum Python Version To Enforce
 minimum-python-version = 2.6

doc/ref/configuration/minion.rst

@@ -81,9 +81,10 @@ The option can can also be set to a list of masters, enabling
 ``ipv6``
 --------
 
-Default: ``False``
+Default: ``None``
 
-Whether the master should be connected over IPv6.
+Whether the master should be connected over IPv6. By default salt minion
+will try to automatically detect IPv6 connectivity to master.
 
 .. code-block:: yaml
 

salt/config/__init__.py

@@ -1062,7 +1062,7 @@ DEFAULT_MINION_OPTS = {
     'mine_interval': 60,
     'ipc_mode': _DFLT_IPC_MODE,
     'ipc_write_buffer': _DFLT_IPC_WBUFFER,
-    'ipv6': False,
+    'ipv6': None,
     'file_buffer_size': 262144,
     'tcp_pub_port': 4510,
     'tcp_pull_port': 4511,

salt/minion.py

@@ -132,7 +132,7 @@ log = logging.getLogger(__name__)
 # 6. Handle publications
 
 
-def resolve_dns(opts, fallback=True):
+def resolve_dns(opts, fallback=True, connect=True):
     '''
     Resolves the master_ip and master_uri options
     '''
@@ -149,13 +149,13 @@ def resolve_dns(opts, fallback=True):
             if opts['master'] == '':
                 raise SaltSystemExit
             ret['master_ip'] = \
-                    salt.utils.dns_check(opts['master'], True, opts['ipv6'])
+                    salt.utils.dns_check(opts['master'], opts['master_port'], True, opts['ipv6'], connect)
         except SaltClientError:
             if opts['retry_dns']:
                 while True:
                     import salt.log
-                    msg = ('Master hostname: \'{0}\' not found. Retrying in {1} '
-                           'seconds').format(opts['master'], opts['retry_dns'])
+                    msg = ('Master hostname: \'{0}\' not found or not responsive. '
+                           'Retrying in {1} seconds').format(opts['master'], opts['retry_dns'])
                     if salt.log.setup.is_console_configured():
                         log.error(msg)
                     else:
@@ -163,7 +163,7 @@ def resolve_dns(opts, fallback=True):
                     time.sleep(opts['retry_dns'])
                     try:
                         ret['master_ip'] = salt.utils.dns_check(
-                            opts['master'], True, opts['ipv6']
+                            opts['master'], opts['master_port'], True, opts['ipv6'], connect
                         )
                         break
                     except SaltClientError:

salt/modules/dockerng.py

@@ -584,6 +584,12 @@ VALID_CREATE_OPTS = {
             'Config': {},
         }
     },
+    'ulimits': {
+        'path': 'HostConfig:Ulimits',
+        'min_docker': (1, 6, 0),
+        'min_docker_py': (1, 2, 0),
+        'default': [],
+    },
 }
 
 
@@ -841,10 +847,10 @@ def _get_client(timeout=None):
                     'Docker machine {0} failed: {1}'.format(docker_machine, exc))
 
         try:
-            __context__['docker.client'] = docker.Client(**client_kwargs)
-        except AttributeError:
             # docker-py 2.0 renamed this client attribute
             __context__['docker.client'] = docker.APIClient(**client_kwargs)
+        except AttributeError:
+            __context__['docker.client'] = docker.Client(**client_kwargs)
 
     # Set a new timeout if one was passed
     if timeout is not None and __context__['docker.client'].timeout != timeout:
@@ -1918,6 +1924,44 @@ def _validate_input(kwargs,
             else:
                 kwargs['labels'] = salt.utils.repack_dictlist(kwargs['labels'])
 
+    def _valid_ulimits():  # pylint: disable=unused-variable
+        '''
+        Must be a string or list of strings with bind mount information
+        '''
+        if kwargs.get('ulimits') is None:
+            # No need to validate
+            return
+        err = (
+            'Invalid ulimits configuration. See the documentation for proper '
+            'usage.'
+        )
+        try:
+            _valid_dictlist('ulimits')
+            # If this was successful then assume the correct API value was
+            # passed on on the CLI and do not proceed with validation.
+            return
+        except SaltInvocationError:
+            pass
+        try:
+            _valid_stringlist('ulimits')
+        except SaltInvocationError:
+            raise SaltInvocationError(err)
+
+        new_ulimits = []
+        for ulimit in kwargs['ulimits']:
+            ulimit_name, comps = ulimit.strip().split('=', 1)
+            try:
+                comps = [int(x) for x in comps.split(':', 1)]
+            except ValueError:
+                raise SaltInvocationError(err)
+            if len(comps) == 1:
+                comps *= 2
+            soft_limit, hard_limit = comps
+            new_ulimits.append({'Name': ulimit_name,
+                                'Soft': soft_limit,
+                                'Hard': hard_limit})
+        kwargs['ulimits'] = new_ulimits
+
     # And now, the actual logic to perform the validation
     if 'docker.docker_version' not in __context__:
         # Have to call this func using the __salt__ dunder (instead of just
@@ -5970,11 +6014,15 @@ def get_client_args():
     except AttributeError:
         try:
             endpoint_config_args = \
-                _argspec(docker.utils.create_endpoint_config).args
+                _argspec(docker.utils.utils.create_endpoint_config).args
         except AttributeError:
-            raise CommandExecutionError(
-                'Failed to get create_host_config argspec'
-            )
+            try:
+                endpoint_config_args = \
+                    _argspec(docker.utils.create_endpoint_config).args
+            except AttributeError:
+                raise CommandExecutionError(
+                    'Failed to get create_endpoint_config argspec'
+                )
 
     for arglist in (config_args, host_config_args, endpoint_config_args):
         try:

salt/modules/openscap.py

@@ -78,6 +78,7 @@ def xccdf(params):
     error = None
     upload_dir = None
     action = None
+    returncode = None
 
     try:
         parser = _ArgumentParser()
@@ -92,14 +93,17 @@ def xccdf(params):
         tempdir = tempfile.mkdtemp()
         proc = Popen(
             shlex.split(cmd), stdout=PIPE, stderr=PIPE, cwd=tempdir)
-        (stdoutdata, stderrdata) = proc.communicate()
+        (stdoutdata, error) = proc.communicate()
         success = _OSCAP_EXIT_CODES_MAP[proc.returncode]
+        returncode = proc.returncode
         if success:
             caller = Caller()
             caller.cmd('cp.push_dir', tempdir)
             shutil.rmtree(tempdir, ignore_errors=True)
             upload_dir = tempdir
-        else:
-            error = stderrdata
 
-    return dict(success=success, upload_dir=upload_dir, error=error)
+    return dict(
+        success=success,
+        upload_dir=upload_dir,
+        error=error,
+        returncode=returncode)

salt/states/dockerng.py

@@ -50,7 +50,7 @@ from salt.modules.dockerng import (
     STOP_TIMEOUT,
     VALID_CREATE_OPTS,
     _validate_input,
-    _get_repo_tag
+    _get_repo_tag,
 )
 # pylint: enable=no-name-in-module,import-error
 import salt.utils
@@ -240,6 +240,7 @@ def _compare(actual, create_kwargs, defaults_from_image):
                 ret.update({item: {'old': actual_ports,
                                    'new': desired_ports}})
             continue
+
         elif item == 'volumes':
             if actual_data is None:
                 actual_data = []
@@ -407,6 +408,7 @@ def _compare(actual, create_kwargs, defaults_from_image):
             # sometimes `[]`. We have to deal with it.
             if bool(actual_data) != bool(data):
                 ret.update({item: {'old': actual_data, 'new': data}})
+
         elif item == 'labels':
             if actual_data is None:
                 actual_data = {}
@@ -433,6 +435,7 @@ def _compare(actual, create_kwargs, defaults_from_image):
             if data != actual_data:
                 ret.update({item: {'old': actual_data, 'new': data}})
             continue
+
         elif item == 'security_opt':
             if actual_data is None:
                 actual_data = []
@@ -448,6 +451,7 @@ def _compare(actual, create_kwargs, defaults_from_image):
                 ret.update({item: {'old': actual_data,
                                    'new': desired_data}})
             continue
+
         elif item in ('cmd', 'command', 'entrypoint'):
             if (actual_data is None and item not in create_kwargs and
                     _image_get(config['image_path'])):
@@ -1521,6 +1525,29 @@ def running(name,
 
             This option requires Docker 1.5.0 or newer.
 
+    ulimits
+        List of ulimits. These limits should be passed in
+        the format ``<ulimit_name>:<soft_limit>:<hard_limit>``, with the hard
+        limit being optional.
+
+        .. code-block:: yaml
+
+            foo:
+              dockerng.running:
+                - image: bar/baz:latest
+                - ulimits: nofile=1024:1024,nproc=60
+
+        Ulimits can be passed as a YAML list instead of a comma-separated list:
+
+        .. code-block:: yaml
+
+            foo:
+              dockerng.running:
+                - image: bar/baz:latest
+                - ulimits:
+                  - nofile=1024:1024
+                  - nproc=60
+
     labels
         Add Metadata to the container. Can be a list of strings/dictionaries
         or a dictionary of strings (keys and values).

salt/states/ssh_known_hosts.py

@@ -26,6 +26,19 @@ import os
 # Import salt libs
 from salt.exceptions import CommandNotFoundError
 
+# Define the state's virtual name
+__virtualname__ = 'ssh_known_hosts'
+
+
+def __virtual__():
+    '''
+    Does not work on Windows, requires ssh module functions
+    '''
+    if salt.utils.is_windows():
+        return False, 'ssh_known_hosts: Does not support Windows'
+
+    return __virtualname__
+
 
 def present(
         name,

salt/transport/zeromq.py

@@ -351,7 +351,7 @@ class AsyncZeroMQPubChannel(salt.transport.mixins.auth.AESPubClientMixin, salt.t
                 zmq.RECONNECT_IVL_MAX, self.opts['recon_max']
             )
 
-        if self.opts['ipv6'] is True and hasattr(zmq, 'IPV4ONLY'):
+        if (self.opts['ipv6'] is True or ':' in self.opts['master_ip']) and hasattr(zmq, 'IPV4ONLY'):
             # IPv6 sockets work for both IPv6 and IPv4 addresses
             self._socket.setsockopt(zmq.IPV4ONLY, 0)
 

salt/utils/__init__.py

@@ -744,10 +744,11 @@ def ip_bracket(addr):
     return addr
 
 
-def dns_check(addr, safe=False, ipv6=False):
+def dns_check(addr, port, safe=False, ipv6=None, connect=True):
     '''
     Return the ip resolved by dns, but do not exit on failure, only raise an
     exception. Obeys system preference for IPv4/6 address resolution.
+    Tries to connect to the address before considering it useful.
     '''
     error = False
     try:
@@ -760,12 +761,30 @@ def dns_check(addr, safe=False, ipv6=False):
         if not hostnames:
             error = True
         else:
-            addr = False
+            resolved = False
             for h in hostnames:
-                if h[0] == socket.AF_INET or (h[0] == socket.AF_INET6 and ipv6):
-                    addr = ip_bracket(h[4][0])
+                if h[0] == socket.AF_INET and ipv6 is True:
+                    continue
+                if h[0] == socket.AF_INET6 and ipv6 is False:
+                    continue
+                if h[0] == socket.AF_INET6 and connect is False and ipv6 is None:
+                    continue
+
+                candidate_addr = ip_bracket(h[4][0])
+
+                if not connect:
+                    resolved = candidate_addr
+
+                s = socket.socket(h[0], socket.SOCK_STREAM)
+                try:
+                    s.connect((candidate_addr.strip('[]'), port))
+                    s.close()
+
+                    resolved = candidate_addr
                     break
-            if not addr:
+                except socket.error:
+                    pass
+            if not resolved:
                 error = True
     except TypeError:
         err = ('Attempt to resolve address \'{0}\' failed. Invalid or unresolveable address').format(addr)
@@ -774,7 +793,7 @@ def dns_check(addr, safe=False, ipv6=False):
         error = True
 
     if error:
-        err = ('DNS lookup of \'{0}\' failed.').format(addr)
+        err = ('DNS lookup or connection check of \'{0}\' failed.').format(addr)
         if safe:
             if salt.log.is_console_configured():
                 # If logging is not configured it also means that either
@@ -783,7 +802,7 @@ def dns_check(addr, safe=False, ipv6=False):
                 log.error(err)
             raise SaltClientError()
         raise SaltSystemExit(code=42, msg=err)
-    return addr
+    return resolved
 
 
 def required_module_list(docstring=None):

tests/unit/config/config_test.py

@@ -358,7 +358,7 @@ class ConfigTestCase(TestCase, integration.AdaptedConfigurationTestCaseMixIn):
         syndic_opts = sconfig.syndic_config(
             syndic_conf_path, minion_conf_path
         )
-        syndic_opts.update(salt.minion.resolve_dns(syndic_opts))
+        syndic_opts.update(salt.minion.resolve_dns(syndic_opts, connect=False))
         root_dir = syndic_opts['root_dir']
         # id & pki dir are shared & so configured on the minion side
         self.assertEqual(syndic_opts['id'], 'minion')

tests/unit/modules/openscap_test.py

@@ -72,7 +72,9 @@ class OpenscapTestCase(TestCase):
             response,
             {
                 'upload_dir': self.random_temp_dir,
-                'error': None, 'success': True
+                'error': '',
+                'success': True,
+                'returncode': 0
             }
         )
 
@@ -80,7 +82,7 @@ class OpenscapTestCase(TestCase):
        'salt.modules.openscap.Popen',
        MagicMock(
            return_value=Mock(
-               **{'returncode': 2, 'communicate.return_value': ('', '')}
+               **{'returncode': 2, 'communicate.return_value': ('', 'some error')}
            )
        )
     )
@@ -111,8 +113,9 @@ class OpenscapTestCase(TestCase):
             response,
             {
                 'upload_dir': self.random_temp_dir,
-                'error': None,
-                'success': True
+                'error': 'some error',
+                'success': True,
+                'returncode': 2
             }
         )
 
@@ -124,7 +127,8 @@ class OpenscapTestCase(TestCase):
             {
                 'error': 'argument --profile is required',
                 'upload_dir': None,
-                'success': False
+                'success': False,
+                'returncode': None
             }
         )
 
@@ -132,7 +136,7 @@ class OpenscapTestCase(TestCase):
        'salt.modules.openscap.Popen',
        MagicMock(
            return_value=Mock(
-               **{'returncode': 2, 'communicate.return_value': ('', '')}
+               **{'returncode': 2, 'communicate.return_value': ('', 'some error')}
            )
        )
     )
@@ -143,8 +147,9 @@ class OpenscapTestCase(TestCase):
             response,
             {
                 'upload_dir': self.random_temp_dir,
-                'error': None,
-                'success': True
+                'error': 'some error',
+                'success': True,
+                'returncode': 2
             }
         )
         expected_cmd = [
@@ -181,19 +186,11 @@ class OpenscapTestCase(TestCase):
             {
                 'upload_dir': None,
                 'error': 'evaluation error',
-                'success': False
+                'success': False,
+                'returncode': 1
             }
         )
 
-    @patch(
-       'salt.modules.openscap.Popen',
-       MagicMock(
-           return_value=Mock(**{
-               'returncode': 1,
-               'communicate.return_value': ('', 'evaluation error')
-           })
-       )
-    )
     def test_openscap_xccdf_eval_fail_not_implemented_action(self):
         response = openscap.xccdf('info {0}'.format(self.policy_file))
 
@@ -202,6 +199,7 @@ class OpenscapTestCase(TestCase):
             {
                 'upload_dir': None,
                 'error': "argument action: invalid choice: 'info' (choose from 'eval')",
-                'success': False
+                'success': False,
+                'returncode': None
             }
         )