Merge branch '2017.7' into cloud-sync-after-install

doc/topics/development/modules/developing.rst

@@ -235,3 +235,17 @@ Defined in: State
 __sdb__
 -------
 Defined in: SDB
+
+
+Additional Globals
+==================
+
+Defined for: Runners, Execution Modules, Wheels
+
+* ``__jid__``: The job ID
+* ``__user__``: The user
+* ``__tag__``: The jid tag
+* ``__jid_event__``: A :py:class:`salt.utils.event.NamespacedEvent`.
+
+:py:class:`NamespacedEvent <salt.utils.event.NamespacedEvent>` defines a single
+method :py:meth:`fire_event <salt.utils.event.NamespacedEvent.fire_event>`, that takes data and tag. The :ref:`Runner docs <runners>` has examples.

requirements/pytest.txt

@@ -5,4 +5,4 @@ pytest-salt == 2018.12.8
 pytest-timeout >= 1.3.3
 pytest-tempdir >= 2018.8.11
 pytest-helpers-namespace >= 2017.11.11
-pytest-salt-from-filenames >= 2019.1.22
+pytest-salt-runtests-bridge >= 2019.1.30

salt/client/mixins.py

@@ -324,7 +324,7 @@ class SyncClientMixin(object):
             print_func=print_func
         )
 
-        # TODO: document these, and test that they exist
+        # TODO: test that they exist
         # TODO: Other things to inject??
         func_globals = {'__jid__': jid,
                         '__user__': data['user'],

salt/cloud/__init__.py

@@ -1338,6 +1338,24 @@ class Cloud(object):
             output['ret'] = action_out
         return output
 
+    @staticmethod
+    def vm_config(name, main, provider, profile, overrides):
+        '''
+        Create vm config.
+
+        :param str name: The name of the vm
+        :param dict main: The main cloud config
+        :param dict provider: The provider config
+        :param dict profile: The profile config
+        :param dict overrides: The vm's config overrides
+        '''
+        vm = main.copy()
+        vm = salt.utils.dictupdate.update(vm, provider)
+        vm = salt.utils.dictupdate.update(vm, profile)
+        vm.update(overrides)
+        vm['name'] = name
+        return vm
+
     def extras(self, extra_):
         '''
         Extra actions
@@ -1424,12 +1442,13 @@ class Cloud(object):
                 ret[name] = {'Error': msg}
                 continue
 
-            vm_ = main_cloud_config.copy()
-            vm_.update(provider_details)
-            vm_.update(profile_details)
-            vm_.update(vm_overrides)
-
-            vm_['name'] = name
+            vm_ = self.vm_config(
+                name,
+                main_cloud_config,
+                provider_details,
+                profile_details,
+                vm_overrides,
+            )
             if self.opts['parallel']:
                 process = multiprocessing.Process(
                     target=self.create,

salt/modules/win_dsc.py

@@ -738,7 +738,7 @@ def set_lcm_config(config_mode=None,
         cmd += '            RefreshFrequencyMins = {0};'.format(refresh_freq)
     if reboot_if_needed is not None:
         if not isinstance(reboot_if_needed, bool):
-            SaltInvocationError('reboot_if_needed must be a boolean value')
+            raise SaltInvocationError('reboot_if_needed must be a boolean value')
         if reboot_if_needed:
             reboot_if_needed = '$true'
         else:

salt/output/nested.py

@@ -34,6 +34,11 @@ from salt.ext.six import string_types
 from salt.utils import get_colors
 import salt.utils.locales
 
+try:
+    from collections.abc import Mapping
+except ImportError:
+    from collections import Mapping
+
 
 class NestDisplay(object):
     '''
@@ -109,7 +114,7 @@ class NestDisplay(object):
                 first_line = False
         elif isinstance(ret, (list, tuple)):
             for ind in ret:
-                if isinstance(ind, (list, tuple, dict)):
+                if isinstance(ind, (list, tuple, Mapping)):
                     out.append(
                         self.ustring(
                             indent,
@@ -117,11 +122,11 @@ class NestDisplay(object):
                             '|_'
                         )
                     )
-                    prefix = '' if isinstance(ind, dict) else '- '
+                    prefix = '' if isinstance(ind, Mapping) else '- '
                     self.display(ind, indent + 2, prefix, out)
                 else:
                     self.display(ind, indent, '- ', out)
-        elif isinstance(ret, dict):
+        elif isinstance(ret, Mapping):
             if indent:
                 out.append(
                     self.ustring(

salt/states/pip_state.py

@@ -27,6 +27,7 @@ import pkg_resources
 
 # Import salt libs
 import salt.utils
+import salt.utils.data
 from salt.version import SaltStackVersion as _SaltStackVersion
 from salt.exceptions import CommandExecutionError, CommandNotFoundError
 
@@ -81,20 +82,6 @@ def __virtual__():
     return False
 
 
-def _find_key(prefix, pip_list):
-    '''
-    Does a case-insensitive match in the pip_list for the desired package.
-    '''
-    try:
-        match = next(
-            iter(x for x in pip_list if x.lower() == prefix.lower())
-        )
-    except StopIteration:
-        return None
-    else:
-        return match
-
-
 def _fulfills_version_spec(version, version_spec):
     '''
     Check version number against version specification info and return a
@@ -210,23 +197,20 @@ def _check_if_installed(prefix, state_pkg_name, version_spec, ignore_installed,
     ret = {'result': False, 'comment': None}
 
     # If we are not passed a pip list, get one:
-    if not pip_list:
-        pip_list = __salt__['pip.list'](prefix, bin_env=bin_env,
-                                        user=user, cwd=cwd,
-                                        env_vars=env_vars, **kwargs)
-
-    # Check if the requested package is already installed.
-    prefix_realname = _find_key(prefix, pip_list)
+    pip_list = salt.utils.data.CaseInsensitiveDict(
+        pip_list or __salt__['pip.list'](prefix, bin_env=bin_env,
+                                         user=user, cwd=cwd,
+                                         env_vars=env_vars, **kwargs)
+    )
 
     # If the package was already installed, check
     # the ignore_installed and force_reinstall flags
-    if ignore_installed is False and prefix_realname is not None:
+    if ignore_installed is False and prefix in pip_list:
         if force_reinstall is False and not upgrade:
             # Check desired version (if any) against currently-installed
             if (
                 any(version_spec) and
-                _fulfills_version_spec(pip_list[prefix_realname],
-                                       version_spec)
+                _fulfills_version_spec(pip_list[prefix], version_spec)
             ) or (not any(version_spec)):
                 ret['result'] = True
                 ret['comment'] = ('Python package {0} was already '
@@ -246,7 +230,7 @@ def _check_if_installed(prefix, state_pkg_name, version_spec, ignore_installed,
                     if 'rc' in spec[1]:
                         include_rc = True
             available_versions = __salt__['pip.list_all_versions'](
-                prefix_realname, bin_env=bin_env, include_alpha=include_alpha,
+                prefix, bin_env=bin_env, include_alpha=include_alpha,
                 include_beta=include_beta, include_rc=include_rc, user=user,
                 cwd=cwd)
             desired_version = ''
@@ -262,9 +246,9 @@ def _check_if_installed(prefix, state_pkg_name, version_spec, ignore_installed,
                 ret['comment'] = ('Python package {0} was already '
                                   'installed and\nthe available upgrade '
                                   'doesn\'t fulfills the version '
-                                  'requirements'.format(prefix_realname))
+                                  'requirements'.format(prefix))
                 return ret
-            if _pep440_version_cmp(pip_list[prefix_realname], desired_version) == 0:
+            if _pep440_version_cmp(pip_list[prefix], desired_version) == 0:
                 ret['result'] = True
                 ret['comment'] = ('Python package {0} was already '
                                   'installed'.format(state_pkg_name))
@@ -898,10 +882,12 @@ def installed(name,
 
                 # Case for packages that are not an URL
                 if prefix:
-                    pipsearch = __salt__['pip.list'](prefix, bin_env,
-                                                     user=user, cwd=cwd,
-                                                     env_vars=env_vars,
-                                                     **kwargs)
+                    pipsearch = salt.utils.data.CaseInsensitiveDict(
+                        __salt__['pip.list'](prefix, bin_env,
+                                             user=user, cwd=cwd,
+                                             env_vars=env_vars,
+                                             **kwargs)
+                    )
 
                     # If we didn't find the package in the system after
                     # installing it report it
@@ -912,12 +898,10 @@ def installed(name,
                             '\'pip.freeze\'.'.format(pkg)
                         )
                     else:
-                        pkg_name = _find_key(prefix, pipsearch)
-                        if pkg_name.lower() in already_installed_packages:
-                            continue
-                        ver = pipsearch[pkg_name]
-                        ret['changes']['{0}=={1}'.format(pkg_name,
-                                                         ver)] = 'Installed'
+                        if prefix in pipsearch \
+                                and prefix.lower() not in already_installed_packages:
+                            ver = pipsearch[prefix]
+                            ret['changes']['{0}=={1}'.format(prefix, ver)] = 'Installed'
                 # Case for packages that are an URL
                 else:
                     ret['changes']['{0}==???'.format(state_name)] = 'Installed'

salt/utils/data.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+'''
+Functions for manipulating, inspecting, or otherwise working with data types
+and data structures.
+'''
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+try:
+    from collections.abc import Mapping, MutableMapping, Sequence
+except ImportError:
+    from collections import Mapping, MutableMapping, Sequence
+
+# Import Salt libs
+from salt.utils.odict import OrderedDict
+
+# Import 3rd-party libs
+from salt.ext import six
+
+
+class CaseInsensitiveDict(MutableMapping):
+    '''
+    Inspired by requests' case-insensitive dict implementation, but works with
+    non-string keys as well.
+    '''
+    def __init__(self, init=None, **kwargs):
+        '''
+        Force internal dict to be ordered to ensure a consistent iteration
+        order, irrespective of case.
+        '''
+        self._data = OrderedDict()
+        self.update(init or {}, **kwargs)
+
+    def __len__(self):
+        return len(self._data)
+
+    def __setitem__(self, key, value):
+        # Store the case-sensitive key so it is available for dict iteration
+        self._data[to_lowercase(key)] = (key, value)
+
+    def __delitem__(self, key):
+        del self._data[to_lowercase(key)]
+
+    def __getitem__(self, key):
+        return self._data[to_lowercase(key)][1]
+
+    def __iter__(self):
+        return (item[0] for item in six.itervalues(self._data))
+
+    def __eq__(self, rval):
+        if not isinstance(rval, Mapping):
+            # Comparing to non-mapping type (e.g. int) is always False
+            return False
+        return dict(self.items_lower()) == dict(CaseInsensitiveDict(rval).items_lower())
+
+    def __repr__(self):
+        return repr(dict(six.iteritems(self)))
+
+    def items_lower(self):
+        '''
+        Returns a generator iterating over keys and values, with the keys all
+        being lowercase.
+        '''
+        return ((key, val[1]) for key, val in six.iteritems(self._data))
+
+    def copy(self):
+        '''
+        Returns a copy of the object
+        '''
+        return CaseInsensitiveDict(six.iteritems(self._data))
+
+
+def __change_case(data, attr, preserve_dict_class=False):
+    try:
+        return getattr(data, attr)()
+    except AttributeError:
+        pass
+
+    data_type = data.__class__
+
+    if isinstance(data, Mapping):
+        return (data_type if preserve_dict_class else dict)(
+            (__change_case(key, attr, preserve_dict_class),
+             __change_case(val, attr, preserve_dict_class))
+            for key, val in six.iteritems(data)
+        )
+    elif isinstance(data, Sequence):
+        return data_type(
+            __change_case(item, attr, preserve_dict_class) for item in data)
+    else:
+        return data
+
+
+def to_lowercase(data, preserve_dict_class=False):
+    return __change_case(data, 'lower', preserve_dict_class)
+
+
+def to_uppercase(data, preserve_dict_class=False):
+    return __change_case(data, 'upper', preserve_dict_class)

setup.py

@@ -15,6 +15,8 @@ import os
 import sys
 import glob
 import time
+import operator
+import platform
 try:
     from urllib2 import urlopen
 except ImportError:
@@ -136,6 +138,74 @@ exec(compile(open(SALT_VERSION).read(), SALT_VERSION, 'exec'))
 
 
 # ----- Helper Functions -------------------------------------------------------------------------------------------->
+
+def _parse_op(op):
+    '''
+    >>> _parse_op('>')
+    'gt'
+    >>> _parse_op('>=')
+    'ge'
+    >>> _parse_op('=>')
+    'ge'
+    >>> _parse_op('=> ')
+    'ge'
+    >>> _parse_op('<')
+    'lt'
+    >>> _parse_op('<=')
+    'le'
+    >>> _parse_op('==')
+    'eq'
+    >>> _parse_op(' <= ')
+    'le'
+    '''
+    op = op.strip()
+    if '>' in op:
+        if '=' in op:
+            return 'ge'
+        else:
+            return 'gt'
+    elif '<' in op:
+        if '=' in op:
+            return 'le'
+        else:
+            return 'lt'
+    elif '!' in op:
+        return 'ne'
+    else:
+        return 'eq'
+
+
+def _parse_ver(ver):
+    '''
+    >>> _parse_ver("'3.4'  # pyzmq 17.1.0 stopped building wheels for python3.4")
+    '3.4'
+    >>> _parse_ver('"3.4"')
+    '3.4'
+    >>> _parse_ver('"2.6.17"')
+    '2.6.17'
+    '''
+    if '#' in ver:
+        ver, _ = ver.split('#', 1)
+        ver = ver.strip()
+    return ver.strip('\'').strip('"')
+
+
+def _check_ver(pyver, op, wanted):
+    '''
+    >>> _check_ver('2.7.15', 'gt', '2.7')
+    True
+    >>> _check_ver('2.7.15', 'gt', '2.7.15')
+    False
+    >>> _check_ver('2.7.15', 'ge', '2.7.15')
+    True
+    >>> _check_ver('2.7.15', 'eq', '2.7.15')
+    True
+    '''
+    pyver = distutils.version.LooseVersion(pyver)
+    wanted = distutils.version.LooseVersion(wanted)
+    return getattr(operator, '__{}__'.format(op))(pyver, wanted)
+
+
 def _parse_requirements_file(requirements_file):
     parsed_requirements = []
     with open(requirements_file) as rfh:
@@ -150,7 +220,16 @@ def _parse_requirements_file(requirements_file):
                 # Python 3 already has futures, installing it will only break
                 # the current python installation whenever futures is imported
                 continue
-            parsed_requirements.append(line)
+            try:
+                pkg, pyverspec = line.rsplit(';', 1)
+            except ValueError:
+                pkg, pyverspec = line, ''
+            pyverspec = pyverspec.strip()
+            if pyverspec:
+                _, op, ver = pyverspec.split(' ', 2)
+                if not _check_ver(platform.python_version(), _parse_op(op), _parse_ver(ver)):
+                    continue
+            parsed_requirements.append(pkg)
     return parsed_requirements
 # <---- Helper Functions ---------------------------------------------------------------------------------------------
 
@@ -362,7 +441,6 @@ class DownloadWindowsDlls(Command):
         if getattr(self.distribution, 'salt_download_windows_dlls', None) is None:
             print('This command is not meant to be called on it\'s own')
             exit(1)
-        import platform
         import pip
         # pip has moved many things to `_internal` starting with pip 10
         if LooseVersion(pip.__version__) < LooseVersion('10.0'):
@@ -932,7 +1010,6 @@ class SaltDistribution(distutils.dist.Distribution):
 
         if IS_WINDOWS_PLATFORM:
             install_requires = _parse_requirements_file(SALT_WINDOWS_REQS)
-
         return install_requires
 
     @property

tests/integration/__init__.py

@@ -181,7 +181,7 @@ class TestDaemon(object):
     '''
     Set up the master and minion daemons, and run related cases
     '''
-    MINIONS_CONNECT_TIMEOUT = MINIONS_SYNC_TIMEOUT = 120
+    MINIONS_CONNECT_TIMEOUT = MINIONS_SYNC_TIMEOUT = 300
 
     def __init__(self, parser):
         self.parser = parser
@@ -217,6 +217,8 @@ class TestDaemon(object):
         if getattr(self.parser.options, 'ssh', False):
             self.prep_ssh()
 
+        self.wait_for_minions(time.time(), self.MINIONS_CONNECT_TIMEOUT)
+
         if self.parser.options.sysinfo:
             try:
                 print_header(
@@ -1321,3 +1323,20 @@ class TestDaemon(object):
     def sync_minion_grains(self, targets, timeout=None):
         salt.utils.appendproctitle('SyncMinionGrains')
         self.sync_minion_modules_('grains', targets, timeout=timeout)
+
+    def wait_for_minions(self, start, timeout, sleep=5):
+        '''
+        Ensure all minions and masters (including sub-masters) are connected.
+        '''
+        while True:
+            try:
+                ret = self.client.run_job('*', 'test.ping')
+            except salt.exceptions.SaltClientError:
+                ret = None
+            if ret and 'minions' not in ret:
+                continue
+            if ret and sorted(ret['minions']) == ['minion', 'sub_minion']:
+                break
+            if time.time() - start >= timeout:
+                raise RuntimeError("Ping Minions Failed")
+            time.sleep(sleep)

tests/integration/modules/test_gem.py

@@ -110,7 +110,7 @@ class GemModuleTest(ModuleCase):
         gem.sources_add
         gem.sources_remove
         '''
-        source = 'http://gems.github.com'
+        source = 'http://gemcutter.org/'
 
         self.run_function('gem.sources_add', [source])
         sources_list = self.run_function('gem.sources_list')

tests/support/case.py

@@ -141,14 +141,14 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
         )
         return self.run_script('salt-ssh', arg_str, with_retcode=with_retcode, catch_stderr=catch_stderr, raw=True)
 
-    def run_run(self, arg_str, with_retcode=False, catch_stderr=False, async=False, timeout=60, config_dir=None):
+    def run_run(self, arg_str, with_retcode=False, catch_stderr=False, async=False, timeout=60, config_dir=None):  # pylint: disable=W8606
         '''
         Execute salt-run
         '''
         arg_str = '-c {0}{async_flag} -t {timeout} {1}'.format(config_dir or self.get_config_dir(),
                                                  arg_str,
                                                  timeout=timeout,
-                                                 async_flag=' --async' if async else '')
+                                                 async_flag=' --async' if async else '')  # pylint: disable=W8606
         return self.run_script('salt-run', arg_str, with_retcode=with_retcode, catch_stderr=catch_stderr)
 
     def run_run_plus(self, fun, *arg, **kwargs):
@@ -243,9 +243,18 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
         script_path = self.get_script_path(script)
         if not os.path.isfile(script_path):
             return False
+        popen_kwargs = popen_kwargs or {}
 
         if salt.utils.is_windows():
             cmd = 'python '
+            if 'cwd' not in popen_kwargs:
+                popen_kwargs['cwd'] = os.getcwd()
+            if 'env' not in popen_kwargs:
+                popen_kwargs['env'] = os.environ.copy()
+                if sys.version_info[0] < 3:
+                    popen_kwargs['env'][b'PYTHONPATH'] = CODE_DIR.encode()
+                else:
+                    popen_kwargs['env']['PYTHONPATH'] = CODE_DIR
         else:
             cmd = 'PYTHONPATH='
             python_path = os.environ.get('PYTHONPATH', None)
@@ -262,7 +271,6 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
 
         tmp_file = tempfile.SpooledTemporaryFile()
 
-        popen_kwargs = popen_kwargs or {}
         popen_kwargs = dict({
             'shell': True,
             'stdout': tmp_file,
@@ -490,14 +498,14 @@ class ShellCase(ShellTestCase, AdaptedConfigurationTestCaseMixin, ScriptPathMixi
                                timeout=timeout,
                                raw=True)
 
-    def run_run(self, arg_str, with_retcode=False, catch_stderr=False, async=False, timeout=60, config_dir=None):
+    def run_run(self, arg_str, with_retcode=False, catch_stderr=False, async=False, timeout=60, config_dir=None):  # pylint: disable=W8606
         '''
         Execute salt-run
         '''
         arg_str = '-c {0}{async_flag} -t {timeout} {1}'.format(config_dir or self.get_config_dir(),
                                                                arg_str,
                                                                timeout=timeout,
-                                                               async_flag=' --async' if async else '')
+                                                               async_flag=' --async' if async else '')  # pylint: disable=W8606
         return self.run_script('salt-run',
                                arg_str,
                                with_retcode=with_retcode,

tests/unit/cloud/__init__.py

@@ -1 +1,70 @@
 # -*- coding: utf-8 -*-
+'''
+    tests.unit.cloud
+    ~~~~~~~~~~~~~~~~
+'''
+from __future__ import absolute_import, print_function, unicode_literals
+from tests.support.unit import TestCase
+
+import salt.cloud
+
+
+class CloudTest(TestCase):
+
+    def test_vm_config_merger(self):
+        '''
+        Validate the vm's config is generated correctly.
+
+        https://github.com/saltstack/salt/issues/49226
+        '''
+        main = {
+            'minion': {'master': '172.31.39.213'},
+            'log_file': 'var/log/salt/cloud.log',
+            'pool_size': 10
+        }
+        provider = {
+            'private_key': 'dwoz.pem',
+            'grains': {'foo1': 'bar', 'foo2': 'bang'},
+            'availability_zone': 'us-west-2b',
+            'driver': 'ec2',
+            'ssh_interface': 'private_ips',
+            'ssh_username': 'admin',
+            'location': 'us-west-2'
+        }
+        profile = {
+            'profile': 'default',
+            'grains': {'meh2': 'bar', 'meh1': 'foo'},
+            'provider': 'ec2-default:ec2',
+            'ssh_username': 'admin',
+            'image': 'ami-0a1fbca0e5b419fd1',
+            'size': 't2.micro'
+        }
+        vm = salt.cloud.Cloud.vm_config(
+            'test_vm',
+            main,
+            provider,
+            profile,
+            {}
+        )
+        self.assertEqual({
+            'minion': {'master': '172.31.39.213'},
+            'log_file': 'var/log/salt/cloud.log',
+            'pool_size': 10,
+            'private_key': 'dwoz.pem',
+            'grains': {
+                'foo1': 'bar',
+                'foo2': 'bang',
+                'meh2': 'bar',
+                'meh1': 'foo',
+            },
+            'availability_zone': 'us-west-2b',
+            'driver': 'ec2',
+            'ssh_interface': 'private_ips',
+            'ssh_username': 'admin',
+            'location': 'us-west-2',
+            'profile': 'default',
+            'provider': 'ec2-default:ec2',
+            'image': 'ami-0a1fbca0e5b419fd1',
+            'size': 't2.micro',
+            'name': 'test_vm',
+        }, vm)

tox.ini

@@ -30,11 +30,8 @@ commands = pytest --rootdir {toxinidir} --log-file={toxinidir}/artifacts/logs/ru
 
 [testenv:runtests-coverage]
 # Add tests/support/coverage to PYTHONPATH in order to get code coverage from subprocesses.
-# Additional, set the COVERAGE_PROCESS_START environment variable so that the coverage library
-# knows it's supposed to track subprocesses.
 setenv =
   PYTHONPATH={toxinidir}/tests/support/coverage
-  COVERAGE_PROCESS_START={toxinidir}/.coveragerc
 commands_pre =
   - coverage erase
 commands =