Merge pull request #45095 from terminalmage/py3-loader

PY3: Make loader ignore .pyc files not in __pycache__

salt/loader.py

@@ -78,15 +78,18 @@ if USE_IMPORTLIB:
 else:
     SUFFIXES = imp.get_suffixes()
 
+BIN_PRE_EXT = '' if six.PY2 \
+    else '.cpython-{0}{1}'.format(sys.version_info.major, sys.version_info.minor)
+
 # Because on the cloud drivers we do `from salt.cloud.libcloudfuncs import *`
 # which simplifies code readability, it adds some unsupported functions into
 # the driver's module scope.
 # We list un-supported functions here. These will be removed from the loaded.
 #  TODO:  remove the need for this cross-module code. Maybe use NotImplemented
 LIBCLOUD_FUNCS_NOT_SUPPORTED = (
-    u'parallels.avail_sizes',
-    u'parallels.avail_locations',
-    u'proxmox.avail_sizes',
+    'parallels.avail_sizes',
+    'parallels.avail_locations',
+    'proxmox.avail_sizes',
 )
 
 # Will be set to pyximport module at runtime if cython is enabled in config.
@@ -987,7 +990,7 @@ def _generate_module(name):
     if name in sys.modules:
         return
 
-    code = u"'''Salt loaded {0} parent module'''".format(name.split('.')[-1])
+    code = "'''Salt loaded {0} parent module'''".format(name.split('.')[-1])
     # ModuleType can't accept a unicode type on PY2
     module = types.ModuleType(str(name))  # future lint: disable=blacklisted-function
     exec(code, module.__dict__)
@@ -1196,6 +1199,7 @@ class LazyLoader(salt.utils.lazy.LazyDict):
         self.suffix_map = {}
         suffix_order = ['']  # local list to determine precedence of extensions
                              # Prefer packages (directories) over modules (single files)!
+
         for (suffix, mode, kind) in SUFFIXES:
             self.suffix_map[suffix] = (suffix, mode, kind)
             suffix_order.append(suffix)
@@ -1224,19 +1228,32 @@ class LazyLoader(salt.utils.lazy.LazyDict):
         self.file_mapping = salt.utils.odict.OrderedDict()
 
         for mod_dir in self.module_dirs:
-            files = []
             try:
-                # Make sure we have a sorted listdir in order to have expectable override results
+                # Make sure we have a sorted listdir in order to have
+                # expectable override results
                 files = sorted(os.listdir(mod_dir))
             except OSError:
                 continue  # Next mod_dir
+            if six.PY3:
+                try:
+                    pycache_files = [
+                        os.path.join('__pycache__', x) for x in
+                        sorted(os.listdir(os.path.join(mod_dir, '__pycache__')))
+                    ]
+                except OSError:
+                    pass
+                else:
+                    pycache_files.extend(files)
+                    files = pycache_files
             for filename in files:
                 try:
-                    if filename.startswith('_'):
+                    dirname, basename = os.path.split(filename)
+                    if basename.startswith('_'):
                         # skip private modules
                         # log messages omitted for obviousness
                         continue  # Next filename
-                    f_noext, ext = os.path.splitext(filename)
+                    f_noext, ext = os.path.splitext(basename)
+                    f_noext = f_noext.replace(BIN_PRE_EXT, '')
                     # make sure it is a suffix we support
                     if ext not in self.suffix_map:
                         continue  # Next filename
@@ -1272,6 +1289,12 @@ class LazyLoader(salt.utils.lazy.LazyDict):
                         if not curr_ext or suffix_order.index(ext) >= suffix_order.index(curr_ext):
                             continue  # Next filename
 
+                    if six.PY3 and not dirname and ext == '.pyc':
+                        # On Python 3, we should only load .pyc files from the
+                        # __pycache__ subdirectory (i.e. when dirname is not an
+                        # empty string).
+                        continue
+
                     # Made it this far - add it
                     self.file_mapping[f_noext] = (fpath, ext)
 

tests/runtests.py

@@ -16,17 +16,6 @@ if os.name == 'nt':
     TESTS_DIR = TESTS_DIR.replace('\\', '\\\\')
 CODE_DIR = os.path.dirname(TESTS_DIR)
 
-if sys.version_info.major == 3:
-    # Clean up any Python 2 byte-compiled files, as they will cause problems
-    # when Python 3 tries to import modules.
-    for root, _, files in os.walk(os.path.dirname(TESTS_DIR)):
-        if os.path.basename(root) == '__pycache__':
-            # Ignore byte-compiled files in Python 3 __pycache__ dirs
-            continue
-        for filename in files:
-            if filename.endswith('.pyc'):
-                os.remove(os.path.join(root, filename))
-
 # Let's inject CODE_DIR so salt is importable if not there already
 if '' in sys.path:
     sys.path.remove('')