Merge branch '2019.2' into rn_update

Gemfile

@@ -4,7 +4,7 @@ source 'https://rubygems.org'
 
 # Point this back at the test-kitchen package after 1.23.3 is relased
 gem 'test-kitchen', '~>1.23.3'
-gem 'kitchen-salt', '~>0.4.1'
+gem 'kitchen-salt', :git => 'https://github.com/s0undt3ch/kitchen-salt.git', :branch => 'features/nox'
 gem 'kitchen-sync'
 gem 'git'
 
@@ -14,7 +14,8 @@ end
 
 group :windows do
   gem 'winrm', '~>2.0'
-  gem 'winrm-fs', '~>1.3.1'
+#  gem 'winrm-fs', '~>1.3.1'
+  gem 'winrm-fs', :git => 'https://github.com/s0undt3ch/winrm-fs.git', :branch => 'hotfix/saltstack-ci'
 end
 
 group :ec2 do

doc/topics/releases/2019.2.0.rst

@@ -8,7 +8,7 @@ Python 2.7 Deprecation
 ======================
 
 In light of Python 2.7 reaching its End of Life (EOL) on Jan 1st 2020,
-Python 2 will be deprecated from SaltStack no earlier then the Sodium
+Python 2 will be deprecated from SaltStack no earlier than the Sodium
 release, that is either the Sodium release or a later release.
 This decision is pending further community discussion.
 

noxfile.py

@@ -23,6 +23,10 @@ import nox
 REPO_ROOT = os.path.abspath(os.path.dirname(__file__))
 SITECUSTOMIZE_DIR = os.path.join(REPO_ROOT, 'tests', 'support', 'coverage')
 
+# We can't just import salt because if this is running under a frozen nox, there
+# will be no salt to import
+IS_WINDOWS = sys.platform.lower().startswith('win')
+
 # Python versions to run against
 _PYTHON_VERSIONS = ('2', '2.7', '3', '3.4', '3.5', '3.6')
 
@@ -41,7 +45,9 @@ def _create_ci_directories():
 
 def _install_requirements(session, *extra_requirements):
     # Install requirements
-    _requirements_files = []
+    _requirements_files = [
+        os.path.join(REPO_ROOT, 'requirements', 'pytest.txt')
+    ]
     if sys.platform.startswith('linux'):
         requirements_files = [
             os.path.join(REPO_ROOT, 'requirements', 'tests.txt')
@@ -49,7 +55,6 @@ def _install_requirements(session, *extra_requirements):
     elif sys.platform.startswith('win'):
         requirements_files = [
             os.path.join(REPO_ROOT, 'pkg', 'windows', 'req.txt'),
-            os.path.join(REPO_ROOT, 'pkg', 'windows', 'req_testing.txt'),
         ]
     elif sys.platform.startswith('darwin'):
         requirements_files = [
@@ -61,6 +66,10 @@ def _install_requirements(session, *extra_requirements):
         if not requirements_files:
             break
         requirements_file = requirements_files.pop(0)
+
+        if requirements_file not in _requirements_files:
+            _requirements_files.append(requirements_file)
+
         session.log('Processing {}'.format(requirements_file))
         with open(requirements_file) as rfh:  # pylint: disable=resource-leakage
             for line in rfh:
@@ -80,8 +89,14 @@ def _install_requirements(session, *extra_requirements):
     if extra_requirements:
         session.install(*extra_requirements)
 
+    if IS_WINDOWS:
+        # Windows hacks :/
+        nox_windows_setup = os.path.join(REPO_ROOT, 'tests', 'support', 'nox-windows-setup.py')
+        session.run('python', nox_windows_setup)
+
 
 def _run_with_coverage(session, *test_cmd):
+    session.install('coverage')
     session.run('coverage', 'erase')
     python_path_env_var = os.environ.get('PYTHONPATH') or None
     if python_path_env_var is None:

salt/cloud/clouds/gce.py

@@ -1828,7 +1828,7 @@ def create_disk(kwargs=None, call=None):
         )
         return False
 
-    if 'size' is None and 'image' is None and 'snapshot' is None:
+    if size is None and image is None and snapshot is None:
         log.error(
             'Must specify image, snapshot, or size.'
         )

salt/modules/github.py

@@ -1256,9 +1256,9 @@ def edit_team(name,
         parameters = {}
         if name is not None:
             parameters['name'] = name
-        if 'description' is not None:
+        if description is not None:
             parameters['description'] = description
-        if 'privacy' is not None:
+        if privacy is not None:
             parameters['privacy'] = privacy
         if permission is not None:
             parameters['permission'] = permission

salt/pillar/pillar_ldap.py

@@ -163,15 +163,17 @@ def _render_template(config_file):
     return template.render(__grains__)
 
 
-def _config(name, conf):
+def _config(name, conf, default=None):
     '''
-    Return a value for 'name' from  the config file options.
+    Return a value for 'name' from the config file options. If the 'name' is
+    not in the config, the 'default' value is returned. This method converts
+    unicode values to str type under python 2.
     '''
     try:
-        value = salt.utils.data.decode(conf[name], to_str=True)
+        value = conf[name]
     except KeyError:
-        value = None
-    return value
+        value = default
+    return salt.utils.data.decode(value, to_str=True)
 
 
 def _result_to_dict(data, result, conf, source):
@@ -285,7 +287,7 @@ def _do_search(conf):
     scope = _config('scope', conf)
     _lists = _config('lists', conf) or []
     _attrs = _config('attrs', conf) or []
-    _dict_key_attr = _config('dict_key_attr', conf) or 'dn'
+    _dict_key_attr = _config('dict_key_attr', conf, 'dn')
     attrs = _lists + _attrs + [_dict_key_attr]
     if not attrs:
         attrs = None

salt/utils/schedule.py

@@ -240,7 +240,7 @@ class Schedule(object):
                             )
                             data['_skip_reason'] = 'maxrunning'
                             data['_skipped'] = True
-                            data['_skip_time'] = now
+                            data['_skipped_time'] = now
                             data['run'] = False
                             return data
         return data
@@ -1378,8 +1378,10 @@ class Schedule(object):
             # Clear these out between runs
             for item in ['_continue',
                          '_error',
+                         '_enabled',
                          '_skipped',
-                         '_skip_reason']:
+                         '_skip_reason',
+                         '_skipped_time']:
                 if item in data:
                     del data[item]
             run = False
@@ -1600,10 +1602,20 @@ class Schedule(object):
             if 'enabled' not in data:
                 data['enabled'] = self.enabled
 
+            # If globally disabled, disable the job
+            if not self.enabled:
+                data['enabled'] = self.enabled
+                data['_skip_reason'] = 'disabled'
+                data['_skipped_time'] = now
+                data['_skipped'] = True
+                run = False
+
             # Job is disabled, set run to False
             if 'enabled' in data and not data['enabled']:
-                log.debug('Job: %s is disabled', job_name)
+                data['_enabled'] = False
                 data['_skip_reason'] = 'disabled'
+                data['_skipped_time'] = now
+                data['_skipped'] = True
                 run = False
 
             miss_msg = ''

salt/utils/systemd.py

@@ -6,6 +6,7 @@ Contains systemd related help files
 from __future__ import absolute_import, print_function, unicode_literals
 import logging
 import os
+import re
 import subprocess
 
 # Import Salt libs
@@ -65,8 +66,8 @@ def version(context=None):
         stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0]
     outstr = salt.utils.stringutils.to_str(stdout)
     try:
-        ret = int(outstr.splitlines()[0].split()[-1])
-    except (IndexError, ValueError):
+        ret = int(re.search(r'\w+ ([0-9]+)', outstr.splitlines()[0]).group(1))
+    except (AttributeError, IndexError, ValueError):
         log.error(
             'Unable to determine systemd version from systemctl '
             '--version, output follows:\n%s', outstr

tests/integration/scheduler/test_eval.py

@@ -506,6 +506,32 @@ class SchedulerEvalTest(ModuleCase, SaltReturnAssertsMixin):
         self.assertNotIn('_last_run', ret)
         self.assertEqual(ret['_skip_reason'], 'disabled')
 
+    def test_eval_global_disabled_job_enabled(self):
+        '''
+        verify that scheduled job does not run
+        '''
+        job_name = 'test_eval_global_disabled'
+        job = {
+          'schedule': {
+            'enabled': False,
+            job_name: {
+              'function': 'test.ping',
+              'when': '11/29/2017 4:00pm',
+              'enabled': True,
+            }
+          }
+        }
+        run_time1 = dateutil_parser.parse('11/29/2017 4:00pm')
+
+        # Add the job to the scheduler
+        self.schedule.opts.update(job)
+
+        # Evaluate 1 second at the run time
+        self.schedule.eval(now=run_time1)
+        ret = self.schedule.job_status(job_name)
+        self.assertNotIn('_last_run', ret)
+        self.assertEqual(ret['_skip_reason'], 'disabled')
+
     def test_eval_run_on_start(self):
         '''
         verify that scheduled job is run when minion starts

tests/support/nox-windows-setup.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+'''
+    tests.support.nox-windows-setup
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    This script is meant to run under the nox virtualenv to take care of required
+    windows procedures
+'''
+# pylint: disable=resource-leakage
+
+from __future__ import absolute_import, print_function, unicode_literals
+import os
+import re
+import sys
+import site
+import shutil
+
+try:
+    import site
+    SITE_PACKAGES = site.getsitepackages()
+    PYTHON_EXECUTABLE_DIRECTORY = os.path.dirname(sys.executable)
+    PYTHON_SCRIPTS_DIR = os.path.join(PYTHON_EXECUTABLE_DIRECTORY, 'Scripts')
+except AttributeError:
+    # The site module does not have the getsitepackages function when running within a virtualenv
+    # But the site-packages directory WILL be on sys.path
+    SITE_PACKAGES = None
+    for entry in sys.path:
+        if 'site-packages' in entry:
+            SITE_PACKAGES = entry
+            break
+    # Under a virtualenv, the python "binary" is under Scripts already.
+    # Well, not the binary, but the Python DLLs
+    PYTHON_EXECUTABLE_DIRECTORY = PYTHON_SCRIPTS_DIR = os.path.dirname(sys.executable)
+
+# Requests is a Salt dependency, it's safe to import, but...
+try:
+    import requests
+    HAS_REQUESTS = True
+except ImportError:
+    HAS_REQUESTS = False
+
+IS_64_BITS = sys.maxsize > 2**32
+SALT_REPO_URL = 'https://repo.saltstack.com/windows/dependencies/{}'.format(IS_64_BITS and 64 or 32)
+DLLS = ("libeay32.dll", "ssleay32.dll", "OpenSSL_License.txt", "msvcr120.dll", "libsodium.dll")
+
+for dll in DLLS:
+    outfile = os.path.join(PYTHON_EXECUTABLE_DIRECTORY, dll)
+    if os.path.exists(outfile):
+        continue
+    src_url = '{}/{}'.format(SALT_REPO_URL, dll)
+    if HAS_REQUESTS:
+        print('Downloading {} to {}'.format(src_url, outfile))
+        request = requests.get(src_url, allow_redirects=True)
+        with open(outfile, 'wb') as wfh:
+            wfh.write(request.content)
+    else:
+        print('ATTENTION: The python requests package is not installed, can\'t download {}'.format(src_url))
+
+PYWIN32_SYSTEM32_DIR = os.path.join(SITE_PACKAGES, 'pywin32_system32')
+if os.path.exists(PYWIN32_SYSTEM32_DIR):
+    for fname in os.listdir(PYWIN32_SYSTEM32_DIR):
+        if not fname.endswith('.dll'):
+            continue
+        spath = os.path.join(PYWIN32_SYSTEM32_DIR, fname)
+        dpath = spath.replace('pywin32_system32', 'win32')
+        print('Moving {} to {}'.format(spath, dpath))
+        shutil.move(spath, dpath)
+
+    print('Deleting {}'.format(PYWIN32_SYSTEM32_DIR))
+    shutil.rmtree(PYWIN32_SYSTEM32_DIR, ignore_errors=True)
+
+
+if os.path.exists(PYTHON_SCRIPTS_DIR):
+    print('Searching for pywin32 scripts to delete')
+    for fname in os.listdir(PYTHON_SCRIPTS_DIR):
+        if not fname.startswith('pywin32_'):
+            continue
+        fpath = os.path.join(PYTHON_SCRIPTS_DIR, fname)
+        print('Deleting {}'.format(fpath))
+        os.unlink(fpath)
+
+
+PYTHONWIN_DIR = os.path.join(SITE_PACKAGES, 'pythonwin')
+if os.path.exists(PYTHONWIN_DIR):
+    print('Deleting {}'.format(PYTHONWIN_DIR))
+    shutil.rmtree(PYTHONWIN_DIR, ignore_errors=True)
+
+PYCRPTO_NT_FILE = os.path.join(SITE_PACKAGES, 'Crypto', 'Random', 'OSRNG', 'nt.py')
+if os.path.exists(PYCRPTO_NT_FILE):
+    with open(PYCRPTO_NT_FILE, 'r') as rfh:
+        contents = rfh.read()
+        new_contents = re.sub(
+            r'^import winrandom$',
+            'from Crypto.Random.OSRNG import winrandom',
+            contents,
+            count=1,
+            flags=re.MULTILINE
+        )
+        if contents != new_contents:
+            print('Patching {}'.format(PYCRPTO_NT_FILE))
+            with open(PYCRPTO_NT_FILE, 'w') as wfh:
+                wfh.write(new_contents)

tests/unit/pillar/test_pillar_ldap.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+
+# Import python libs
+from __future__ import absolute_import, print_function, unicode_literals
+
+from tests.support.unit import TestCase
+import salt.utils.stringutils
+
+
+from salt.pillar.pillar_ldap import _config
+
+
+class LdapPillarTestCase(TestCase):
+
+    def test__config_returns_str(self):
+        conf = {'foo': 'bar'}
+        assert _config('foo', conf) == salt.utils.stringutils.to_str('bar')
+
+    def test__conf_defaults_to_none(self):
+        conf = {'foo': 'bar'}
+        assert _config('bang', conf) is None
+
+    def test__conf_returns_str_from_unicode_default(self):
+        conf = {'foo': 'bar'}
+        default = salt.utils.stringutils.to_unicode('bam')
+        assert _config('bang', conf, default) == salt.utils.stringutils.to_str('bam')

tests/unit/utils/test_systemd.py

@@ -100,6 +100,27 @@ class SystemdTestCase(TestCase):
             self.assertTrue(_systemd.version(context))
             self.assertEqual(context, {'salt.utils.systemd.version': _version})
 
+    def test_version_generated_from_git_describe(self):
+        '''
+        Test with version string matching versions generated by git describe
+        in systemd. This feature is used in systemd>=241.
+        '''
+        with patch('subprocess.Popen') as popen_mock:
+            _version = 241
+            output = 'systemd {0} ({0}.0-0-dist)\n-SYSVINIT'.format(_version)
+            popen_mock.return_value = Mock(
+                communicate=lambda *args, **kwargs: (output, None),
+                pid=lambda: 12345,
+                retcode=0
+            )
+
+            # Test without context dict passed
+            self.assertEqual(_systemd.version(), _version)
+            # Test that context key is set when context dict is passed
+            context = {}
+            self.assertTrue(_systemd.version(context))
+            self.assertEqual(context, {'salt.utils.systemd.version': _version})
+
     def test_version_return_from_context(self):
         '''
         Test that the context data is returned when present. To ensure we're