Merge branch '2018.3' into 'fluorine'

Conflicts:
  - tests/integration/runners/test_cache.py
  - tests/support/case.py

doc/ref/configuration/master.rst

@@ -1782,6 +1782,25 @@ the master will drop the request and the minion's key will remain accepted.
 
     allow_minion_key_revoke: False
 
+.. conf_master:: optimization_order
+
+``optimization_order``
+----------------------
+
+Default: ``[0, 1, 2]``
+
+In cases where Salt is distributed without .py files, this option determines
+the priority of optimization level(s) Salt's module loader should prefer.
+
+.. note::
+    This option is only supported on Python 3.5+.
+
+.. code-block:: yaml
+
+    optimization_order:
+      - 2
+      - 0
+      - 1
 
 Master Large Scale Tuning Settings
 ==================================

doc/ref/configuration/minion.rst

@@ -1511,6 +1511,25 @@ Specifies which keys are examined by
         - GlobalIPv6Address
         - IPv6Gateway
 
+.. conf_minion:: optimization_order
+
+``optimization_order``
+----------------------
+
+Default: ``[0, 1, 2]``
+
+In cases where Salt is distributed without .py files, this option determines
+the priority of optimization level(s) Salt's module loader should prefer.
+
+.. note::
+    This option is only supported on Python 3.5+.
+
+.. code-block:: yaml
+
+    optimization_order:
+      - 2
+      - 0
+      - 1
 
 Minion Execution Module Management
 ==================================

salt/config/__init__.py

@@ -312,6 +312,9 @@ VALID_OPTS = {
     # The type of hashing algorithm to use when doing file comparisons
     'hash_type': six.string_types,
 
+    # Order of preference for optimized .pyc files (PY3 only)
+    'optimization_order': list,
+
     # Refuse to load these modules
     'disable_modules': list,
 
@@ -1346,6 +1349,7 @@ DEFAULT_MINION_OPTS = {
     'gitfs_disable_saltenv_mapping': False,
     'unique_jid': False,
     'hash_type': 'sha256',
+    'optimization_order': [0, 1, 2],
     'disable_modules': [],
     'disable_returners': [],
     'whitelist_modules': [],
@@ -1680,6 +1684,7 @@ DEFAULT_MASTER_OPTS = {
     'fileserver_verify_config': True,
     'max_open_files': 100000,
     'hash_type': 'sha256',
+    'optimization_order': [0, 1, 2],
     'conf_file': os.path.join(salt.syspaths.CONFIG_DIR, 'master'),
     'open_mode': False,
     'auto_accept': False,

salt/grains/core.py

@@ -1357,6 +1357,7 @@ _OS_NAME_MAP = {
     'poky': 'Poky',
     'manjaro': 'Manjaro',
     'manjarolin': 'Manjaro',
+    'univention': 'Univention',
     'antergos': 'Antergos',
     'sles': 'SUSE',
     'void': 'Void',
@@ -1417,6 +1418,7 @@ _OS_FAMILY_MAP = {
     'GCEL': 'Debian',
     'Linaro': 'Debian',
     'elementary OS': 'Debian',
+    'Univention': 'Debian',
     'ScientificLinux': 'RedHat',
     'Raspbian': 'Debian',
     'Devuan': 'Debian',

salt/loader.py

@@ -8,6 +8,7 @@ plugin interfaces used by Salt.
 # Import python libs
 from __future__ import absolute_import, print_function, unicode_literals
 import os
+import re
 import sys
 import time
 import logging
@@ -83,8 +84,8 @@ if USE_IMPORTLIB:
 else:
     SUFFIXES = imp.get_suffixes()
 
-BIN_PRE_EXT = '' if six.PY2 \
-    else '.cpython-{0}{1}'.format(sys.version_info.major, sys.version_info.minor)
+PY3_PRE_EXT = \
+    re.compile(r'\.cpython-{0}{1}(\.opt-[1-9])?'.format(*sys.version_info[:2]))
 
 # Because on the cloud drivers we do `from salt.cloud.libcloudfuncs import *`
 # which simplifies code readability, it adds some unsupported functions into
@@ -1137,6 +1138,15 @@ class LazyLoader(salt.utils.lazy.LazyDict):
             )
         )
 
+        # A map of suffix to description for imp
+        self.suffix_map = {}
+        # A list to determine precedence of extensions
+        # Prefer packages (directories) over modules (single files)!
+        self.suffix_order = ['']
+        for (suffix, mode, kind) in SUFFIXES:
+            self.suffix_map[suffix] = (suffix, mode, kind)
+            self.suffix_order.append(suffix)
+
         self._lock = threading.RLock()
         self._refresh_file_mapping()
 
@@ -1210,14 +1220,6 @@ class LazyLoader(salt.utils.lazy.LazyDict):
         refresh the mapping of the FS on disk
         '''
         # map of suffix to description for imp
-        self.suffix_map = {}
-        suffix_order = ['']  # local list to determine precedence of extensions
-                             # Prefer packages (directories) over modules (single files)!
-
-        for (suffix, mode, kind) in SUFFIXES:
-            self.suffix_map[suffix] = (suffix, mode, kind)
-            suffix_order.append(suffix)
-
         if self.opts.get('cython_enable', True) is True:
             try:
                 global pyximport
@@ -1241,11 +1243,25 @@ class LazyLoader(salt.utils.lazy.LazyDict):
         # The files are added in order of priority, so order *must* be retained.
         self.file_mapping = salt.utils.odict.OrderedDict()
 
+        opt_match = []
+
+        def _replace_pre_ext(obj):
+            '''
+            Hack so we can get the optimization level that we replaced (if
+            any) out of the re.sub call below. We use a list here because
+            it is a persistent data structure that we will be able to
+            access after re.sub is called.
+            '''
+            opt_match.append(obj)
+            return ''
+
         for mod_dir in self.module_dirs:
             try:
                 # Make sure we have a sorted listdir in order to have
                 # expectable override results
-                files = sorted(os.listdir(mod_dir))
+                files = sorted(
+                    x for x in os.listdir(mod_dir) if x != '__pycache__'
+                )
             except OSError:
                 continue  # Next mod_dir
             if six.PY3:
@@ -1257,8 +1273,8 @@ class LazyLoader(salt.utils.lazy.LazyDict):
                 except OSError:
                     pass
                 else:
-                    pycache_files.extend(files)
-                    files = pycache_files
+                    files.extend(pycache_files)
+
             for filename in files:
                 try:
                     dirname, basename = os.path.split(filename)
@@ -1267,7 +1283,30 @@ class LazyLoader(salt.utils.lazy.LazyDict):
                         # log messages omitted for obviousness
                         continue  # Next filename
                     f_noext, ext = os.path.splitext(basename)
-                    f_noext = f_noext.replace(BIN_PRE_EXT, '')
+                    if six.PY3:
+                        f_noext = PY3_PRE_EXT.sub(_replace_pre_ext, f_noext)
+                        try:
+                            opt_level = int(
+                                opt_match.pop().group(1).rsplit('-', 1)[-1]
+                            )
+                        except (AttributeError, IndexError, ValueError):
+                            # No regex match or no optimization level matched
+                            opt_level = 0
+                        try:
+                            opt_index = self.opts['optimization_order'].index(opt_level)
+                        except KeyError:
+                            log.trace(
+                                'Disallowed optimization level %d for module '
+                                'name \'%s\', skipping. Add %d to the '
+                                '\'optimization_order\' config option if you '
+                                'do not want to ignore this optimization '
+                                'level.', opt_level, f_noext, opt_level
+                            )
+                            continue
+                    else:
+                        # Optimization level not reflected in filename on PY2
+                        opt_index = 0
+
                     # make sure it is a suffix we support
                     if ext not in self.suffix_map:
                         continue  # Next filename
@@ -1282,7 +1321,7 @@ class LazyLoader(salt.utils.lazy.LazyDict):
                     if ext == '':
                         # is there something __init__?
                         subfiles = os.listdir(fpath)
-                        for suffix in suffix_order:
+                        for suffix in self.suffix_order:
                             if '' == suffix:
                                 continue  # Next suffix (__init__ must have a suffix)
                             init_file = '__init__{0}'.format(suffix)
@@ -1291,17 +1330,29 @@ class LazyLoader(salt.utils.lazy.LazyDict):
                         else:
                             continue  # Next filename
 
-                    if f_noext in self.file_mapping:
+                    try:
                         curr_ext = self.file_mapping[f_noext][1]
-                        #log.debug("****** curr_ext={0} ext={1} suffix_order={2}".format(curr_ext, ext, suffix_order))
+                        curr_opt_index = self.file_mapping[f_noext][2]
+                    except KeyError:
+                        pass
+                    else:
                         if '' in (curr_ext, ext) and curr_ext != ext:
                             log.error(
                                 'Module/package collision: \'%s\' and \'%s\'',
                                 fpath,
                                 self.file_mapping[f_noext][0]
                             )
-                        if not curr_ext or suffix_order.index(ext) >= suffix_order.index(curr_ext):
-                            continue  # Next filename
+
+                        if six.PY3 and ext == '.pyc' and curr_ext == '.pyc':
+                            # Check the optimization level
+                            if opt_index >= curr_opt_index:
+                                # Module name match, but a higher-priority
+                                # optimization level was already matched, skipping.
+                                continue
+                        elif not curr_ext or self.suffix_order.index(ext) >= self.suffix_order.index(curr_ext):
+                            # Match found but a higher-priorty match already
+                            # exists, so skip this.
+                            continue
 
                     if six.PY3 and not dirname and ext == '.pyc':
                         # On Python 3, we should only load .pyc files from the
@@ -1310,13 +1361,13 @@ class LazyLoader(salt.utils.lazy.LazyDict):
                         continue
 
                     # Made it this far - add it
-                    self.file_mapping[f_noext] = (fpath, ext)
+                    self.file_mapping[f_noext] = (fpath, ext, opt_index)
 
                 except OSError:
                     continue
         for smod in self.static_modules:
             f_noext = smod.split('.')[-1]
-            self.file_mapping[f_noext] = (smod, '.o')
+            self.file_mapping[f_noext] = (smod, '.o', 0)
 
     def clear(self):
         '''
@@ -1385,7 +1436,7 @@ class LazyLoader(salt.utils.lazy.LazyDict):
 
     def _load_module(self, name):
         mod = None
-        fpath, suffix = self.file_mapping[name]
+        fpath, suffix = self.file_mapping[name][:2]
         self.loaded_files.add(name)
         fpath_dirname = os.path.dirname(fpath)
         try:

salt/returners/highstate_return.py

@@ -187,8 +187,7 @@ def _generate_html_table(data, out, level=0, extra_style=''):
             else:
                 new_extra_style = extra_style
             if len(subdata) == 1:
-                name = subdata.keys()[0]
-                value = subdata.values()[0]
+                name, value = next(six.iteritems(subdata))
                 print('<tr style="{0}">'.format(
                     _lookup_style('tr', [row_style])
                 ), file=out)

salt/states/netyang.py

@@ -80,7 +80,7 @@ def managed(name,
             *models,
             **kwargs):
     '''
-    Manage the device configuration given the input data strucuted
+    Manage the device configuration given the input data structured
     according to the YANG models.
 
     data

tests/integration/cloud/providers/test_vultr.py

@@ -20,6 +20,7 @@ from salt.ext import six
 # Create the cloud instance name to be used throughout the tests
 INSTANCE_NAME = generate_random_name('CLOUD-TEST-')
 PROVIDER_NAME = 'vultr'
+TIMEOUT = 500
 
 
 class VultrTest(ShellCase):
@@ -143,20 +144,20 @@ class VultrTest(ShellCase):
         '''
         # check if instance with salt installed returned
         try:
-            create_vm = self.run_cloud('-p vultr-test {0}'.format(INSTANCE_NAME), timeout=500)
+            create_vm = self.run_cloud('-p vultr-test {0}'.format(INSTANCE_NAME), timeout=800)
             self.assertIn(
                 INSTANCE_NAME,
                 [i.strip() for i in create_vm]
             )
             self.assertNotIn('Failed to start', six.text_type(create_vm))
         except AssertionError:
-            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=500)
+            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=TIMEOUT)
             raise
 
         # Vultr won't let us delete an instance less than 5 minutes old.
         time.sleep(420)
         # delete the instance
-        results = self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=500)
+        results = self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=TIMEOUT)
         try:
             self.assertIn(
                 'True',
@@ -172,6 +173,6 @@ class VultrTest(ShellCase):
         # If we exceed 6 minutes and the instance is still there, quit
         ct = 0
         while ct < 12 and INSTANCE_NAME in [i.strip() for i in self.run_cloud('--query')]:
-            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=500)
+            self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=TIMEOUT)
             time.sleep(30)
             ct = ct + 1

tests/integration/minion/test_pillar.py

@@ -36,6 +36,7 @@ GPG_SLS = os.path.join(PILLAR_BASE, 'gpg.sls')
 DEFAULT_OPTS = {
     'cachedir': os.path.join(TMP, 'rootdir', 'cache'),
     'config_dir': TMP_CONF_DIR,
+    'optimization_order': [0, 1, 2],
     'extension_modules': os.path.join(TMP,
                                       'test-decrypt-pillar',
                                       'extmods'),

tests/integration/modules/test_grains.py

@@ -165,16 +165,23 @@ class GrainsAppendTestCase(ModuleCase):
 
     def test_grains_append_val_already_present(self):
         '''
-        Tests the return of a grains.append call when the value is already present in the grains list.
+        Tests the return of a grains.append call when the value is already
+        present in the grains list.
         '''
-        messaging = 'The val {0} was already in the list salttesting-grain-key'.format(self.GRAIN_VAL)
+        msg = 'The val {0} was already in the list ' \
+              'salttesting-grain-key'.format(self.GRAIN_VAL)
 
         # First, make sure the test grain is present
         self.run_function('grains.append', [self.GRAIN_KEY, self.GRAIN_VAL])
 
         # Now try to append again
         ret = self.run_function('grains.append', [self.GRAIN_KEY, self.GRAIN_VAL])
-        self.assertEqual(messaging, ret)
+        if not ret or isinstance(ret, dict):
+            # Sleep for a bit, sometimes the second "append" runs too quickly
+            time.sleep(5)
+            ret = self.run_function('grains.append', [self.GRAIN_KEY, self.GRAIN_VAL])
+
+        assert msg == ret
 
     def test_grains_append_val_is_list(self):
         '''

tests/integration/modules/test_service.py

@@ -129,7 +129,7 @@ class ServiceModuleTest(ModuleCase):
         if tuple(self.run_function('grains.item', ['osrelease_info'])['osrelease_info']) == (14, 0o4) and not systemd:
             # currently upstart does not have a mechanism to report if disabling a service fails if does not exist
             self.assertTrue(self.run_function('service.disable', [srv_name]))
-        elif self.run_function('grains.item', ['osfullname'])['osfullname'] == 'Debian' and \
+        elif self.run_function('grains.item', ['os'])['os'] == 'Debian' and \
              self.run_function('grains.item', ['osmajorrelease'])['osmajorrelease'] < 9 and systemd:
              # currently disabling a service via systemd that does not exist
              # on Debian 8 results in a True return code

tests/integration/netapi/test_client.py

@@ -3,10 +3,11 @@
 # Import Python libs
 from __future__ import absolute_import, print_function, unicode_literals
 import os
+import time
 
 # Import Salt Testing libs
-from tests.support.unit import TestCase
 from tests.support.paths import TMP_CONF_DIR
+from tests.support.unit import TestCase, skipIf
 
 # Import Salt libs
 import salt.config
@@ -84,6 +85,8 @@ class NetapiClientTest(TestCase):
         self.assertTrue(set(['master.pem', 'master.pub']).issubset(set(ret['data']['return']['local'])))
 
     def test_wheel_async(self):
+        # Give this test a little breathing room
+        time.sleep(3)
         low = {'client': 'wheel_async', 'fun': 'key.list_all'}
         low.update(self.eauth_creds)
 
@@ -91,6 +94,7 @@ class NetapiClientTest(TestCase):
         self.assertIn('jid', ret)
         self.assertIn('tag', ret)
 
+    @skipIf(True, 'This is not testing anything. Skipping for now.')
     def test_runner(self):
         # TODO: fix race condition in init of event-- right now the event class
         # will finish init even if the underlying zmq socket hasn't connected yet
@@ -102,6 +106,7 @@ class NetapiClientTest(TestCase):
 
         ret = self.netapi.run(low)
 
+    @skipIf(True, 'This is not testing anything. Skipping for now.')
     def test_runner_async(self):
         low = {'client': 'runner', 'fun': 'cache.grains'}
         low.update(self.eauth_creds)

tests/integration/states/test_compiler.py

@@ -9,6 +9,16 @@ from __future__ import absolute_import, unicode_literals, print_function
 # Import Salt Testing libs
 from tests.support.case import ModuleCase
 
+# Import Salt libs
+import salt.utils.platform
+
+# Import 3rd-Party libs
+HAS_LSB_RELEASE = True
+try:
+    import lsb_release
+except ImportError:
+    HAS_LSB_RELEASE = False
+
 
 class CompileTest(ModuleCase):
     '''
@@ -27,6 +37,11 @@ class CompileTest(ModuleCase):
         Test when we have an error in a execution module
         called by jinja
         '''
+        if salt.utils.platform.is_linux() and HAS_LSB_RELEASE:
+            release = lsb_release.get_distro_information()
+            if release.get('ID') == 'Debian' and int(release.get('RELEASE', '0')[0]) < 9:
+                self.skipTest('This test is flaky on Debian 8. Skipping.')
+
         ret = self.run_function('state.sls', ['issue-10010'])
         self.assertTrue(
             ', in jinja_error' in ret[0].strip())

tests/support/gitfs.py

@@ -331,11 +331,14 @@ class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin):
         '''
         cachedir = tempfile.mkdtemp(dir=TMP)
         self.addCleanup(shutil.rmtree, cachedir, ignore_errors=True)
-        ext_pillar_opts = salt.utils.yaml.safe_load(
-            ext_pillar_conf.format(
-                cachedir=cachedir,
-                extmods=os.path.join(cachedir, 'extmods'),
-                **self.ext_opts
+        ext_pillar_opts = {'optimization_order': [0, 1, 2]}
+        ext_pillar_opts.update(
+            salt.utils.yaml.safe_load(
+                ext_pillar_conf.format(
+                    cachedir=cachedir,
+                    extmods=os.path.join(cachedir, 'extmods'),
+                    **self.ext_opts
+                )
             )
         )
         with patch.dict(git_pillar.__opts__, ext_pillar_opts):

tests/unit/cloud/test_map_conf.py

@@ -115,6 +115,7 @@ class MapConfTest(TestCase):
                 patch('salt.cloud.Map.read', MagicMock(return_value=EXAMPLE_MAP)):
             self.maxDiff = None
             opts = {'extension_modules': '/var/cache/salt/master/extmods',
+                    'optimization_order': [0, 1, 2],
                     'providers': EXAMPLE_PROVIDERS, 'profiles': EXAMPLE_PROFILES}
             cloud_map = salt.cloud.Map(opts)
 

tests/unit/loader/test_loader.py

@@ -8,9 +8,11 @@
 
 # Import Python libs
 from __future__ import absolute_import, print_function, unicode_literals
+import compileall
 import inspect
 import logging
 import tempfile
+import textwrap
 import shutil
 import os
 import collections
@@ -32,7 +34,15 @@ from salt.ext import six
 from salt.ext.six.moves import range
 # pylint: enable=no-name-in-module,redefined-builtin
 
-from salt.loader import LazyLoader, _module_dirs, grains, utils, proxy, minion_mods
+from salt.loader import (
+    LazyLoader,
+    USE_IMPORTLIB,
+    _module_dirs,
+    grains,
+    utils,
+    proxy,
+    minion_mods
+)
 
 log = logging.getLogger(__name__)
 
@@ -955,3 +965,131 @@ class LazyLoaderDeepSubmodReloadingTest(TestCase):
                 self.update_lib(lib)
                 self.loader.clear()
                 self._verify_libs()
+
+
+class LazyLoaderOptimizationOrderTest(TestCase):
+    '''
+    Test the optimization order priority in the loader (PY3)
+    '''
+    module_name = 'lazyloadertest'
+    module_content = textwrap.dedent('''\
+        # -*- coding: utf-8 -*-
+        from __future__ import absolute_import
+
+        def test():
+            return True
+        ''')
+
+    @classmethod
+    def setUpClass(cls):
+        cls.opts = salt.config.minion_config(None)
+        cls.opts['grains'] = grains(cls.opts)
+
+    def setUp(self):
+        # Setup the module
+        self.module_dir = tempfile.mkdtemp(dir=TMP)
+        self.module_file = os.path.join(self.module_dir,
+                                        '{0}.py'.format(self.module_name))
+
+    def _get_loader(self, order=None):
+        opts = copy.deepcopy(self.opts)
+        if order is not None:
+            opts['optimization_order'] = order
+        # Return a loader
+        return LazyLoader([self.module_dir], opts, tag='module')
+
+    def _get_module_filename(self):
+        # The act of referencing the loader entry forces the module to be
+        # loaded by the LazyDict.
+        mod_fullname = self.loader[next(iter(self.loader))].__module__
+        return sys.modules[mod_fullname].__file__
+
+    def _expected(self, optimize=0):
+        if six.PY3:
+            return 'lazyloadertest.cpython-{0}{1}{2}.pyc'.format(
+                sys.version_info[0],
+                sys.version_info[1],
+                '' if not optimize else '.opt-{0}'.format(optimize)
+            )
+        else:
+            return 'lazyloadertest.pyc'
+
+    def _write_module_file(self):
+        with salt.utils.files.fopen(self.module_file, 'w') as fh:
+            fh.write(self.module_content)
+            fh.flush()
+            os.fsync(fh.fileno())
+
+    def _byte_compile(self):
+        if USE_IMPORTLIB:
+            # Skip this check as "optimize" is unique to PY3's compileall
+            # module, and this will be a false error when Pylint is run on
+            # Python 2.
+            # pylint: disable=unexpected-keyword-arg
+            compileall.compile_file(self.module_file, quiet=1, optimize=0)
+            compileall.compile_file(self.module_file, quiet=1, optimize=1)
+            compileall.compile_file(self.module_file, quiet=1, optimize=2)
+            # pylint: enable=unexpected-keyword-arg
+        else:
+            compileall.compile_file(self.module_file, quiet=1)
+
+    def _test_optimization_order(self, order):
+        self._write_module_file()
+        self._byte_compile()
+
+        # Clean up the original file so that we can be assured we're only
+        # loading the byte-compiled files(s).
+        os.remove(self.module_file)
+
+        self.loader = self._get_loader(order)
+        filename = self._get_module_filename()
+        basename = os.path.basename(filename)
+        assert basename == self._expected(order[0]), basename
+
+        if not USE_IMPORTLIB:
+            # We are only testing multiple optimization levels on Python 3.5+
+            return
+
+        # Remove the file and make a new loader. We should now load the
+        # byte-compiled file with an optimization level matching the 2nd
+        # element of the order list.
+        os.remove(filename)
+        self.loader = self._get_loader(order)
+        filename = self._get_module_filename()
+        basename = os.path.basename(filename)
+        assert basename == self._expected(order[1]), basename
+
+        # Remove the file and make a new loader. We should now load the
+        # byte-compiled file with an optimization level matching the 3rd
+        # element of the order list.
+        os.remove(filename)
+        self.loader = self._get_loader(order)
+        filename = self._get_module_filename()
+        basename = os.path.basename(filename)
+        assert basename == self._expected(order[2]), basename
+
+    def test_optimization_order(self):
+        '''
+        Test the optimization_order config param
+        '''
+        self._test_optimization_order([0, 1, 2])
+        self._test_optimization_order([0, 2, 1])
+        if USE_IMPORTLIB:
+            # optimization_order only supported on Python 3.5+, earlier
+            # releases only support unoptimized .pyc files.
+            self._test_optimization_order([1, 2, 0])
+            self._test_optimization_order([1, 0, 2])
+            self._test_optimization_order([2, 0, 1])
+            self._test_optimization_order([2, 1, 0])
+
+    def test_load_source_file(self):
+        '''
+        Make sure that .py files are preferred over .pyc files
+        '''
+        self._write_module_file()
+        self._byte_compile()
+        self.loader = self._get_loader()
+        filename = self._get_module_filename()
+        basename = os.path.basename(filename)
+        expected = 'lazyloadertest.py' if six.PY3 else 'lazyloadertest.pyc'
+        assert basename == expected, basename

tests/unit/output/test_highstate_out.py

@@ -27,6 +27,7 @@ class JsonTestCase(TestCase, LoaderModuleMockMixin):
             highstate: {
                 '__opts__': {
                     'extension_modules': '',
+                    'optimization_order': [0, 1, 2],
                     'color': False,
                 }
             }

tests/unit/pillar/test_file_tree.py

@@ -74,6 +74,7 @@ class FileTreePillarTestCase(TestCase, LoaderModuleMockMixin):
                     },
                     'pillarenv': 'base',
                     'nodegroups': {'test-group': [MINION_ID]},
+                    'optimization_order': [0, 1, 2],
                     'file_buffer_size': 262144,
                     'file_roots': {'base': '', 'dev': '', 'parent': ''},
                     'extension_modules': '',

tests/unit/returners/test_smtp_return.py

@@ -32,7 +32,25 @@ class SMTPReturnerTestCase(TestCase, LoaderModuleMockMixin):
     Test SMTP returner
     '''
     def setup_loader_modules(self):
-        return {smtp: {}}
+        return {
+            smtp: {
+                '__opts__': {
+                    'extension_modules': '',
+                    'optimization_order': [0, 1, 2],
+                    'renderer': 'jinja|yaml',
+                    'renderer_blacklist': [],
+                    'renderer_whitelist': [],
+                    'file_roots': {},
+                    'pillar_roots': {},
+                    'cachedir': '/',
+                    'master_uri': 'tcp://127.0.0.1:4505',
+                    'pki_dir': '/',
+                    'keysize': 2048,
+                    'id': 'test',
+                    '__role': 'minion',
+                }
+            }
+        }
 
     def _test_returner(self, mocked_smtplib):  # pylint: disable=unused-argument
         '''
@@ -62,25 +80,11 @@ class SMTPReturnerTestCase(TestCase, LoaderModuleMockMixin):
 
     if HAS_GNUPG:
         def test_returner(self):
-            with patch.dict(smtp.__opts__, {'extension_modules': '',
-                                            'renderer': 'jinja|yaml',
-                                            'renderer_blacklist': [],
-                                            'renderer_whitelist': [],
-                                            'file_roots': {},
-                                            'pillar_roots': {},
-                                            'cachedir': '/'}), \
-                    patch('salt.returners.smtp_return.gnupg'), \
+            with patch('salt.returners.smtp_return.gnupg'), \
                     patch('salt.returners.smtp_return.smtplib.SMTP') as mocked_smtplib:
                 self._test_returner(mocked_smtplib)
 
     else:
         def test_returner(self):
-            with patch.dict(smtp.__opts__, {'extension_modules': '',
-                                            'renderer': 'jinja|yaml',
-                                            'renderer_blacklist': [],
-                                            'renderer_whitelist': [],
-                                            'file_roots': {},
-                                            'pillar_roots': {},
-                                            'cachedir': '/'}), \
-                    patch('salt.returners.smtp_return.smtplib.SMTP') as mocked_smtplib:
+            with patch('salt.returners.smtp_return.smtplib.SMTP') as mocked_smtplib:
                 self._test_returner(mocked_smtplib)

tests/unit/runners/test_winrepo.py

@@ -90,6 +90,7 @@ class WinrepoTest(TestCase, LoaderModuleMockMixin):
             winrepo: {
                 '__opts__': {
                     'winrepo_cachefile': 'winrepo.p',
+                    'optimization_order': [0, 1, 2],
                     'renderer': 'yaml',
                     'renderer_blacklist': [],
                     'renderer_whitelist': [],

tests/unit/templates/test_jinja.py

@@ -14,6 +14,7 @@ import tempfile
 # Import Salt Testing libs
 from tests.support.unit import skipIf, TestCase
 from tests.support.case import ModuleCase
+from tests.support.helpers import flaky
 from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock, Mock
 from tests.support.paths import TMP_CONF_DIR
 
@@ -1078,6 +1079,7 @@ class TestCustomExtensions(TestCase):
                                      dict(opts=self.local_opts, saltenv='test', salt=self.local_salt))
         self.assertEqual(rendered, '16777216')
 
+    @flaky
     def test_http_query(self):
         '''
         Test the `http_query` Jinja filter.

tests/unit/test_pillar.py

@@ -35,6 +35,7 @@ class PillarTestCase(TestCase):
     def test_pillarenv_from_saltenv(self):
         with patch('salt.pillar.compile_template') as compile_template:
             opts = {
+                'optimization_order': [0, 1, 2],
                 'renderer': 'json',
                 'renderer_blacklist': [],
                 'renderer_whitelist': [],
@@ -59,6 +60,7 @@ class PillarTestCase(TestCase):
 
     def test_ext_pillar_no_extra_minion_data_val_dict(self):
         opts = {
+            'optimization_order': [0, 1, 2],
             'renderer': 'json',
             'renderer_blacklist': [],
             'renderer_whitelist': [],
@@ -99,6 +101,7 @@ class PillarTestCase(TestCase):
 
     def test_ext_pillar_no_extra_minion_data_val_list(self):
         opts = {
+            'optimization_order': [0, 1, 2],
             'renderer': 'json',
             'renderer_blacklist': [],
             'renderer_whitelist': [],
@@ -139,6 +142,7 @@ class PillarTestCase(TestCase):
 
     def test_ext_pillar_no_extra_minion_data_val_elem(self):
         opts = {
+            'optimization_order': [0, 1, 2],
             'renderer': 'json',
             'renderer_blacklist': [],
             'renderer_whitelist': [],
@@ -177,6 +181,7 @@ class PillarTestCase(TestCase):
 
     def test_ext_pillar_with_extra_minion_data_val_dict(self):
         opts = {
+            'optimization_order': [0, 1, 2],
             'renderer': 'json',
             'renderer_blacklist': [],
             'renderer_whitelist': [],
@@ -217,6 +222,7 @@ class PillarTestCase(TestCase):
 
     def test_ext_pillar_with_extra_minion_data_val_list(self):
         opts = {
+            'optimization_order': [0, 1, 2],
             'renderer': 'json',
             'renderer_blacklist': [],
             'renderer_whitelist': [],
@@ -257,6 +263,7 @@ class PillarTestCase(TestCase):
 
     def test_ext_pillar_with_extra_minion_data_val_elem(self):
         opts = {
+            'optimization_order': [0, 1, 2],
             'renderer': 'json',
             'renderer_blacklist': [],
             'renderer_whitelist': [],
@@ -297,6 +304,7 @@ class PillarTestCase(TestCase):
 
     def test_dynamic_pillarenv(self):
         opts = {
+            'optimization_order': [0, 1, 2],
             'renderer': 'json',
             'renderer_blacklist': [],
             'renderer_whitelist': [],
@@ -311,6 +319,7 @@ class PillarTestCase(TestCase):
 
     def test_ignored_dynamic_pillarenv(self):
         opts = {
+            'optimization_order': [0, 1, 2],
             'renderer': 'json',
             'renderer_blacklist': [],
             'renderer_whitelist': [],
@@ -326,6 +335,7 @@ class PillarTestCase(TestCase):
     def test_malformed_pillar_sls(self, mock_list_states):
         with patch('salt.pillar.compile_template') as compile_template:
             opts = {
+                'optimization_order': [0, 1, 2],
                 'renderer': 'json',
                 'renderer_blacklist': [],
                 'renderer_whitelist': [],
@@ -437,6 +447,7 @@ class PillarTestCase(TestCase):
 
     def test_includes_override_sls(self):
         opts = {
+            'optimization_order': [0, 1, 2],
             'renderer': 'json',
             'renderer_blacklist': [],
             'renderer_whitelist': [],
@@ -501,6 +512,7 @@ class PillarTestCase(TestCase):
         with patch('salt.pillar.salt.fileclient.get_file_client', autospec=True) as get_file_client, \
                 patch('salt.pillar.salt.minion.Matcher') as Matcher:  # autospec=True disabled due to py3 mock bug
             opts = {
+                'optimization_order': [0, 1, 2],
                 'renderer': 'yaml',
                 'renderer_blacklist': [],
                 'renderer_whitelist': [],

tests/unit/utils/test_sdb.py

@@ -23,6 +23,7 @@ TEMP_DATABASE_FILE = os.path.join(RUNTIME_VARS.TMP, 'test_sdb.sqlite')
 
 SDB_OPTS = {
             'extension_modules': '',
+            'optimization_order': [0, 1, 2],
             'test_sdb_data': {
                 'driver': 'sqlite3',
                 'database': TEMP_DATABASE_FILE,