mirror of
https://github.com/saltstack/salt.git
synced 2025-04-16 01:30:20 +00:00
002aa88a97
Without allow_unicode=True, unicode characters are processed through the str representer and on Python 2 are dumped as a Unicode code point (i.e. a literal \u0414). This commit makes allow_unicode=True the default in our salt.utils.yamlloader.safe_dump() helper. It also adds a new salt.utils.yamlloader.dump() helper which wraps yaml.dump() and also makes allow_unicode=True the default. To make importing and using our custom yaml loader/dumper easier, a convenience module called salt.utils.yaml has been added, which does a wildcard import from both salt.utils.yamldumper and salt.utils.yamlloader. Refs to yaml.load/dump and yaml.safe_load/safe_dump have been updated to salt.utils.yaml, to ensure that unicode is handled properly.
503 lines
17 KiB
Python
503 lines
17 KiB
Python
# -*- coding: utf-8 -*-
|
|
'''
|
|
Functions used for CLI argument handling
|
|
'''
|
|
|
|
# Import python libs
|
|
from __future__ import absolute_import
|
|
import copy
|
|
import fnmatch
|
|
import inspect
|
|
import re
|
|
import shlex
|
|
|
|
# Import salt libs
|
|
from salt.exceptions import SaltInvocationError
|
|
from salt.ext import six
|
|
from salt.ext.six.moves import zip # pylint: disable=import-error,redefined-builtin
|
|
import salt.utils.data
|
|
import salt.utils.jid
|
|
import salt.utils.versions
|
|
import salt.utils.yaml
|
|
|
|
|
|
if six.PY3:
|
|
KWARG_REGEX = re.compile(r'^([^\d\W][\w.-]*)=(?!=)(.*)$', re.UNICODE)
|
|
else:
|
|
KWARG_REGEX = re.compile(r'^([^\d\W][\w.-]*)=(?!=)(.*)$')
|
|
|
|
|
|
def clean_kwargs(**kwargs):
|
|
'''
|
|
Return a dict without any of the __pub* keys (or any other keys starting
|
|
with a dunder) from the kwargs dict passed into the execution module
|
|
functions. These keys are useful for tracking what was used to invoke
|
|
the function call, but they may not be desirable to have if passing the
|
|
kwargs forward wholesale.
|
|
'''
|
|
ret = {}
|
|
for key, val in six.iteritems(kwargs):
|
|
if not key.startswith('__'):
|
|
ret[key] = val
|
|
return ret
|
|
|
|
|
|
def invalid_kwargs(invalid_kwargs, raise_exc=True):
|
|
'''
|
|
Raise a SaltInvocationError if invalid_kwargs is non-empty
|
|
'''
|
|
if invalid_kwargs:
|
|
if isinstance(invalid_kwargs, dict):
|
|
new_invalid = [
|
|
'{0}={1}'.format(x, y)
|
|
for x, y in six.iteritems(invalid_kwargs)
|
|
]
|
|
invalid_kwargs = new_invalid
|
|
msg = (
|
|
'The following keyword arguments are not valid: {0}'
|
|
.format(', '.join(invalid_kwargs))
|
|
)
|
|
if raise_exc:
|
|
raise SaltInvocationError(msg)
|
|
else:
|
|
return msg
|
|
|
|
|
|
def condition_input(args, kwargs):
|
|
'''
|
|
Return a single arg structure for the publisher to safely use
|
|
'''
|
|
ret = []
|
|
for arg in args:
|
|
if (six.PY3 and isinstance(arg, six.integer_types) and salt.utils.jid.is_jid(str(arg))) or \
|
|
(six.PY2 and isinstance(arg, long)): # pylint: disable=incompatible-py3-code
|
|
ret.append(str(arg))
|
|
else:
|
|
ret.append(arg)
|
|
if isinstance(kwargs, dict) and kwargs:
|
|
kw_ = {'__kwarg__': True}
|
|
for key, val in six.iteritems(kwargs):
|
|
kw_[key] = val
|
|
return ret + [kw_]
|
|
return ret
|
|
|
|
|
|
def parse_input(args, condition=True, no_parse=None):
|
|
'''
|
|
Parse out the args and kwargs from a list of input values. Optionally,
|
|
return the args and kwargs without passing them to condition_input().
|
|
|
|
Don't pull args with key=val apart if it has a newline in it.
|
|
'''
|
|
if no_parse is None:
|
|
no_parse = ()
|
|
_args = []
|
|
_kwargs = {}
|
|
for arg in args:
|
|
if isinstance(arg, six.string_types):
|
|
arg_name, arg_value = parse_kwarg(arg)
|
|
if arg_name:
|
|
_kwargs[arg_name] = yamlify_arg(arg_value) \
|
|
if arg_name not in no_parse \
|
|
else arg_value
|
|
else:
|
|
_args.append(yamlify_arg(arg))
|
|
elif isinstance(arg, dict):
|
|
# Yes, we're popping this key off and adding it back if
|
|
# condition_input is called below, but this is the only way to
|
|
# gracefully handle both CLI and API input.
|
|
if arg.pop('__kwarg__', False) is True:
|
|
_kwargs.update(arg)
|
|
else:
|
|
_args.append(arg)
|
|
else:
|
|
_args.append(arg)
|
|
if condition:
|
|
return condition_input(_args, _kwargs)
|
|
return _args, _kwargs
|
|
|
|
|
|
def parse_kwarg(string_):
|
|
'''
|
|
Parses the string and looks for the following kwarg format:
|
|
|
|
"{argument name}={argument value}"
|
|
|
|
For example: "my_message=Hello world"
|
|
|
|
Returns the kwarg name and value, or (None, None) if the regex was not
|
|
matched.
|
|
'''
|
|
try:
|
|
return KWARG_REGEX.match(string_).groups()
|
|
except AttributeError:
|
|
return None, None
|
|
|
|
|
|
def yamlify_arg(arg):
|
|
'''
|
|
yaml.safe_load the arg
|
|
'''
|
|
if not isinstance(arg, six.string_types):
|
|
return arg
|
|
|
|
if arg.strip() == '':
|
|
# Because YAML loads empty strings as None, we return the original string
|
|
# >>> import yaml
|
|
# >>> yaml.load('') is None
|
|
# True
|
|
# >>> yaml.load(' ') is None
|
|
# True
|
|
return arg
|
|
|
|
elif '_' in arg and all([x in '0123456789_' for x in arg.strip()]):
|
|
return arg
|
|
|
|
try:
|
|
# Explicit late import to avoid circular import. DO NOT MOVE THIS.
|
|
import salt.utils.yaml
|
|
original_arg = arg
|
|
if '#' in arg:
|
|
# Only yamlify if it parses into a non-string type, to prevent
|
|
# loss of content due to # as comment character
|
|
parsed_arg = salt.utils.yaml.safe_load(arg)
|
|
if isinstance(parsed_arg, six.string_types) or parsed_arg is None:
|
|
return arg
|
|
return parsed_arg
|
|
if arg == 'None':
|
|
arg = None
|
|
else:
|
|
arg = salt.utils.yaml.safe_load(arg)
|
|
|
|
if isinstance(arg, dict):
|
|
# dicts must be wrapped in curly braces
|
|
if (isinstance(original_arg, six.string_types) and
|
|
not original_arg.startswith('{')):
|
|
return original_arg
|
|
else:
|
|
return arg
|
|
|
|
elif arg is None \
|
|
or isinstance(arg, (list, float, six.integer_types, six.string_types)):
|
|
# yaml.safe_load will load '|' as '', don't let it do that.
|
|
if arg == '' and original_arg in ('|',):
|
|
return original_arg
|
|
# yaml.safe_load will treat '#' as a comment, so a value of '#'
|
|
# will become None. Keep this value from being stomped as well.
|
|
elif arg is None and original_arg.strip().startswith('#'):
|
|
return original_arg
|
|
else:
|
|
return arg
|
|
else:
|
|
# we don't support this type
|
|
return original_arg
|
|
except Exception:
|
|
# In case anything goes wrong...
|
|
return original_arg
|
|
|
|
|
|
if six.PY3:
|
|
from collections import namedtuple # pylint: disable=wrong-import-position,wrong-import-order
|
|
|
|
_ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults')
|
|
|
|
def _getargspec(func):
|
|
'''
|
|
Python 3 wrapper for inspect.getargsspec
|
|
|
|
inspect.getargsspec is deprecated and will be removed in Python 3.6.
|
|
'''
|
|
args, varargs, varkw, defaults, kwonlyargs, _, ann = \
|
|
inspect.getfullargspec(func) # pylint: disable=no-member
|
|
if kwonlyargs or ann:
|
|
raise ValueError('Function has keyword-only arguments or annotations'
|
|
', use getfullargspec() API which can support them')
|
|
return _ArgSpec(args, varargs, varkw, defaults)
|
|
|
|
|
|
def get_function_argspec(func, is_class_method=None):
|
|
'''
|
|
A small wrapper around getargspec that also supports callable classes
|
|
:param is_class_method: Pass True if you are sure that the function being passed
|
|
is a class method. The reason for this is that on Python 3
|
|
``inspect.ismethod`` only returns ``True`` for bound methods,
|
|
while on Python 2, it returns ``True`` for bound and unbound
|
|
methods. So, on Python 3, in case of a class method, you'd
|
|
need the class to which the function belongs to be instantiated
|
|
and this is not always wanted.
|
|
'''
|
|
if not callable(func):
|
|
raise TypeError('{0} is not a callable'.format(func))
|
|
|
|
if six.PY2:
|
|
if is_class_method is True:
|
|
aspec = inspect.getargspec(func)
|
|
del aspec.args[0] # self
|
|
elif inspect.isfunction(func):
|
|
aspec = inspect.getargspec(func)
|
|
elif inspect.ismethod(func):
|
|
aspec = inspect.getargspec(func)
|
|
del aspec.args[0] # self
|
|
elif isinstance(func, object):
|
|
aspec = inspect.getargspec(func.__call__)
|
|
del aspec.args[0] # self
|
|
else:
|
|
raise TypeError(
|
|
'Cannot inspect argument list for \'{0}\''.format(func)
|
|
)
|
|
else:
|
|
if is_class_method is True:
|
|
aspec = _getargspec(func)
|
|
del aspec.args[0] # self
|
|
elif inspect.isfunction(func):
|
|
aspec = _getargspec(func) # pylint: disable=redefined-variable-type
|
|
elif inspect.ismethod(func):
|
|
aspec = _getargspec(func)
|
|
del aspec.args[0] # self
|
|
elif isinstance(func, object):
|
|
aspec = _getargspec(func.__call__)
|
|
del aspec.args[0] # self
|
|
else:
|
|
raise TypeError(
|
|
'Cannot inspect argument list for \'{0}\''.format(func)
|
|
)
|
|
return aspec
|
|
|
|
|
|
def shlex_split(s, **kwargs):
|
|
'''
|
|
Only split if variable is a string
|
|
'''
|
|
if isinstance(s, six.string_types):
|
|
return shlex.split(s, **kwargs)
|
|
else:
|
|
return s
|
|
|
|
|
|
def arg_lookup(fun, aspec=None):
|
|
'''
|
|
Return a dict containing the arguments and default arguments to the
|
|
function.
|
|
'''
|
|
ret = {'kwargs': {}}
|
|
if aspec is None:
|
|
aspec = get_function_argspec(fun)
|
|
if aspec.defaults:
|
|
ret['kwargs'] = dict(zip(aspec.args[::-1], aspec.defaults[::-1]))
|
|
ret['args'] = [arg for arg in aspec.args if arg not in ret['kwargs']]
|
|
return ret
|
|
|
|
|
|
def argspec_report(functions, module=''):
|
|
'''
|
|
Pass in a functions dict as it is returned from the loader and return the
|
|
argspec function signatures
|
|
'''
|
|
ret = {}
|
|
if '*' in module or '.' in module:
|
|
for fun in fnmatch.filter(functions, module):
|
|
try:
|
|
aspec = get_function_argspec(functions[fun])
|
|
except TypeError:
|
|
# this happens if not callable
|
|
continue
|
|
|
|
args, varargs, kwargs, defaults = aspec
|
|
|
|
ret[fun] = {}
|
|
ret[fun]['args'] = args if args else None
|
|
ret[fun]['defaults'] = defaults if defaults else None
|
|
ret[fun]['varargs'] = True if varargs else None
|
|
ret[fun]['kwargs'] = True if kwargs else None
|
|
|
|
else:
|
|
# "sys" should just match sys without also matching sysctl
|
|
module_dot = module + '.'
|
|
|
|
for fun in functions:
|
|
if fun.startswith(module_dot):
|
|
try:
|
|
aspec = get_function_argspec(functions[fun])
|
|
except TypeError:
|
|
# this happens if not callable
|
|
continue
|
|
|
|
args, varargs, kwargs, defaults = aspec
|
|
|
|
ret[fun] = {}
|
|
ret[fun]['args'] = args if args else None
|
|
ret[fun]['defaults'] = defaults if defaults else None
|
|
ret[fun]['varargs'] = True if varargs else None
|
|
ret[fun]['kwargs'] = True if kwargs else None
|
|
|
|
return ret
|
|
|
|
|
|
def split_input(val):
|
|
'''
|
|
Take an input value and split it into a list, returning the resulting list
|
|
'''
|
|
if isinstance(val, list):
|
|
return val
|
|
try:
|
|
return [x.strip() for x in val.split(',')]
|
|
except AttributeError:
|
|
return [x.strip() for x in str(val).split(',')]
|
|
|
|
|
|
def test_mode(**kwargs):
|
|
'''
|
|
Examines the kwargs passed and returns True if any kwarg which matching
|
|
"Test" in any variation on capitalization (i.e. "TEST", "Test", "TeSt",
|
|
etc) contains a True value (as determined by salt.utils.data.is_true).
|
|
'''
|
|
# Once is_true is moved, remove this import and fix the ref below
|
|
import salt.utils
|
|
for arg, value in six.iteritems(kwargs):
|
|
try:
|
|
if arg.lower() == 'test' and salt.utils.data.is_true(value):
|
|
return True
|
|
except AttributeError:
|
|
continue
|
|
return False
|
|
|
|
|
|
def format_call(fun,
|
|
data,
|
|
initial_ret=None,
|
|
expected_extra_kws=(),
|
|
is_class_method=None):
|
|
'''
|
|
Build the required arguments and keyword arguments required for the passed
|
|
function.
|
|
|
|
:param fun: The function to get the argspec from
|
|
:param data: A dictionary containing the required data to build the
|
|
arguments and keyword arguments.
|
|
:param initial_ret: The initial return data pre-populated as dictionary or
|
|
None
|
|
:param expected_extra_kws: Any expected extra keyword argument names which
|
|
should not trigger a :ref:`SaltInvocationError`
|
|
:param is_class_method: Pass True if you are sure that the function being passed
|
|
is a class method. The reason for this is that on Python 3
|
|
``inspect.ismethod`` only returns ``True`` for bound methods,
|
|
while on Python 2, it returns ``True`` for bound and unbound
|
|
methods. So, on Python 3, in case of a class method, you'd
|
|
need the class to which the function belongs to be instantiated
|
|
and this is not always wanted.
|
|
:returns: A dictionary with the function required arguments and keyword
|
|
arguments.
|
|
'''
|
|
ret = initial_ret is not None and initial_ret or {}
|
|
|
|
ret['args'] = []
|
|
ret['kwargs'] = {}
|
|
|
|
aspec = get_function_argspec(fun, is_class_method=is_class_method)
|
|
|
|
arg_data = arg_lookup(fun, aspec)
|
|
args = arg_data['args']
|
|
kwargs = arg_data['kwargs']
|
|
|
|
# Since we WILL be changing the data dictionary, let's change a copy of it
|
|
data = data.copy()
|
|
|
|
missing_args = []
|
|
|
|
for key in kwargs:
|
|
try:
|
|
kwargs[key] = data.pop(key)
|
|
except KeyError:
|
|
# Let's leave the default value in place
|
|
pass
|
|
|
|
while args:
|
|
arg = args.pop(0)
|
|
try:
|
|
ret['args'].append(data.pop(arg))
|
|
except KeyError:
|
|
missing_args.append(arg)
|
|
|
|
if missing_args:
|
|
used_args_count = len(ret['args']) + len(args)
|
|
args_count = used_args_count + len(missing_args)
|
|
raise SaltInvocationError(
|
|
'{0} takes at least {1} argument{2} ({3} given)'.format(
|
|
fun.__name__,
|
|
args_count,
|
|
args_count > 1 and 's' or '',
|
|
used_args_count
|
|
)
|
|
)
|
|
|
|
ret['kwargs'].update(kwargs)
|
|
|
|
if aspec.keywords:
|
|
# The function accepts **kwargs, any non expected extra keyword
|
|
# arguments will made available.
|
|
for key, value in six.iteritems(data):
|
|
if key in expected_extra_kws:
|
|
continue
|
|
ret['kwargs'][key] = value
|
|
|
|
# No need to check for extra keyword arguments since they are all
|
|
# **kwargs now. Return
|
|
return ret
|
|
|
|
# Did not return yet? Lets gather any remaining and unexpected keyword
|
|
# arguments
|
|
extra = {}
|
|
for key, value in six.iteritems(data):
|
|
if key in expected_extra_kws:
|
|
continue
|
|
extra[key] = copy.deepcopy(value)
|
|
|
|
# We'll be showing errors to the users until Salt Fluorine comes out, after
|
|
# which, errors will be raised instead.
|
|
salt.utils.versions.warn_until(
|
|
'Fluorine',
|
|
'It\'s time to start raising `SaltInvocationError` instead of '
|
|
'returning warnings',
|
|
# Let's not show the deprecation warning on the console, there's no
|
|
# need.
|
|
_dont_call_warnings=True
|
|
)
|
|
|
|
if extra:
|
|
# Found unexpected keyword arguments, raise an error to the user
|
|
if len(extra) == 1:
|
|
msg = '\'{0[0]}\' is an invalid keyword argument for \'{1}\''.format(
|
|
list(extra.keys()),
|
|
ret.get(
|
|
# In case this is being called for a state module
|
|
'full',
|
|
# Not a state module, build the name
|
|
'{0}.{1}'.format(fun.__module__, fun.__name__)
|
|
)
|
|
)
|
|
else:
|
|
msg = '{0} and \'{1}\' are invalid keyword arguments for \'{2}\''.format(
|
|
', '.join(['\'{0}\''.format(e) for e in extra][:-1]),
|
|
list(extra.keys())[-1],
|
|
ret.get(
|
|
# In case this is being called for a state module
|
|
'full',
|
|
# Not a state module, build the name
|
|
'{0}.{1}'.format(fun.__module__, fun.__name__)
|
|
)
|
|
)
|
|
|
|
# Return a warning to the user explaining what's going on
|
|
ret.setdefault('warnings', []).append(
|
|
'{0}. If you were trying to pass additional data to be used '
|
|
'in a template context, please populate \'context\' with '
|
|
'\'key: value\' pairs. Your approach will work until Salt '
|
|
'Fluorine is out.{1}'.format(
|
|
msg,
|
|
'' if 'full' not in ret else ' Please update your state files.'
|
|
)
|
|
)
|
|
|
|
# Lets pack the current extra kwargs as template context
|
|
ret.setdefault('context', {}).update(extra)
|
|
return ret
|