saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

PY3: Support different optimization levels

Browse source 
This uses the optimization_order config option to control which module
is loaded when multiple optimization levels have been installed.
This commit is contained in:
Erik Johnson 2018-07-17 15:27:57 -05:00
parent 6fc8da5bab
commit c3622933c1
No known key found for this signature in database
GPG key ID: 5E5583C437808F3F
1 changed files with 58 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

67
salt/loader.py
View file

@ -8,6 +8,7 @@ plugin interfaces used by Salt.
# Import python libs
from __future__ import absolute_import
import os
import re
import sys
import time
import logging
@ -79,8 +80,8 @@ if USE_IMPORTLIB:
else:
SUFFIXES = imp.get_suffixes()
BIN_PRE_EXT = '' if six.PY2 \
else '.cpython-{0}{1}'.format(sys.version_info.major, sys.version_info.minor)
PY3_PRE_EXT = \
re.compile(r'\.cpython-{0}{1}(\.opt-[1-9])?'.format(*sys.version_info[:2]))
# Because on the cloud drivers we do `from salt.cloud.libcloudfuncs import *`
# which simplifies code readability, it adds some unsupported functions into
@ -1207,6 +1208,17 @@ class LazyLoader(salt.utils.lazy.LazyDict):
# The files are added in order of priority, so order *must* be retained.
self.file_mapping = salt.utils.odict.OrderedDict()
opt_match = []
def _replace_pre_ext(obj):
'''
Hack so we can get the optimization level that we replaced (if
any) out of the re.sub call below. We use a list here because
it is a persistent data structure that we will be able to
access after re.sub is called.
'''
opt_match.append(obj)
return ''
for mod_dir in self.module_dirs:
try:
# Make sure we have a sorted listdir in order to have
@ -1225,6 +1237,7 @@ class LazyLoader(salt.utils.lazy.LazyDict):
else:
pycache_files.extend(files)
files = pycache_files
for filename in files:
try:
dirname, basename = os.path.split(filename)
@ -1233,7 +1246,30 @@ class LazyLoader(salt.utils.lazy.LazyDict):
# log messages omitted for obviousness
continue # Next filename
f_noext, ext = os.path.splitext(basename)
f_noext = f_noext.replace(BIN_PRE_EXT, '')
if six.PY3:
f_noext = PY3_PRE_EXT.sub(_replace_pre_ext, f_noext)
try:
opt_level = int(
opt_match.pop().group(1).rsplit('-', 1)[-1]
)
except (AttributeError, IndexError, ValueError):
# No regex match or no optimization level matched
opt_level = 0
try:
opt_index = self.opts['optimization_order'].index(opt_level)
except KeyError:
log.trace(
'Disallowed optimization level %d for module '
'name \'%s\', skipping. Add %d to the '
'\'optimization_order\' config option if you '
'do not want to ignore this optimization '
'level.', opt_level, f_noext, opt_level
)
continue
else:
# Optimization level not reflected in filename on PY2
opt_index = 0
# make sure it is a suffix we support
if ext not in self.suffix_map:
continue # Next filename
@ -1258,16 +1294,29 @@ class LazyLoader(salt.utils.lazy.LazyDict):
else:
continue # Next filename
if f_noext in self.file_mapping:
try:
curr_ext = self.file_mapping[f_noext][1]
curr_opt_index = self.file_mapping[f_noext][2]
except KeyError:
pass
else:
if '' in (curr_ext, ext) and curr_ext != ext:
log.error(
'Module/package collision: \'%s\' and \'%s\'',
fpath,
self.file_mapping[f_noext][0]
)
if not curr_ext or suffix_order.index(ext) >= suffix_order.index(curr_ext):
continue # Next filename
if six.PY3 and ext == '.pyc' and curr_ext == '.pyc':
# Check the optimization level
if opt_index >= curr_opt_index:
# Module name match, but a higher-priority
# optimization level was already matched, skipping.
continue
elif not curr_ext or suffix_order.index(ext) >= suffix_order.index(curr_ext):
# Match found but a higher-priorty match already
# exists, so skip this.
continue
if six.PY3 and not dirname and ext == '.pyc':
# On Python 3, we should only load .pyc files from the
@ -1276,13 +1325,13 @@ class LazyLoader(salt.utils.lazy.LazyDict):
continue
# Made it this far - add it
self.file_mapping[f_noext] = (fpath, ext)
self.file_mapping[f_noext] = (fpath, ext, opt_index)
except OSError:
continue
for smod in self.static_modules:
f_noext = smod.split('.')[-1]
self.file_mapping[f_noext] = (smod, '.o')
self.file_mapping[f_noext] = (smod, '.o', 0)
def clear(self):
'''
@ -1351,7 +1400,7 @@ class LazyLoader(salt.utils.lazy.LazyDict):
def _load_module(self, name):
mod = None
fpath, suffix = self.file_mapping[name]
fpath, suffix = self.file_mapping[name][:2]
self.loaded_files.add(name)
fpath_dirname = os.path.dirname(fpath)
try: