saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #41679 from terminalmage/get-top-file-envs

Browse source 
Prevent unnecessary duplicate pillar compilation
This commit is contained in:
Mike Place 2017-06-21 10:32:42 -05:00 • committed by GitHub
commit 4d0f5c433d
12 changed files with 317 additions and 221 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
salt/daemons/masterapi.py
View file

@ -716,7 +716,7 @@ class RemoteFuncs(object):
load.get('saltenv', load.get('env')),
load.get('ext'),
self.mminion.functions,
pillar=load.get('pillar_override', {}))
pillar_override=load.get('pillar_override', {}))
pillar_dirs = {}
data = pillar.compile_pillar(pillar_dirs=pillar_dirs)
if self.opts.get('minion_data_cache', False):

2
salt/master.py
View file

@ -1339,7 +1339,7 @@ class AESFuncs(object):
load['id'],
load.get('saltenv', load.get('env')),
ext=load.get('ext'),
pillar=load.get('pillar_override', {}),
pillar_override=load.get('pillar_override', {}),
pillarenv=load.get('pillarenv'))
data = pillar.compile_pillar(pillar_dirs=pillar_dirs)
self.fs_.update_opts()

2
salt/modules/cmdmod.py
View file

@ -211,7 +211,7 @@ def _gather_pillar(pillarenv, pillar_override):
__grains__,
__opts__['id'],
__opts__['environment'],
pillar=pillar_override,
pillar_override=pillar_override,
pillarenv=pillarenv
)
ret = pillar.compile_pillar()

2
salt/modules/cp.py
View file

@ -48,7 +48,7 @@ def _gather_pillar(pillarenv, pillar_override):
__grains__,
__opts__['id'],
__opts__['environment'],
pillar=pillar_override,
pillar_override=pillar_override,
pillarenv=pillarenv
)
ret = pillar.compile_pillar()

2
salt/modules/dockerng.py
View file

@ -5617,7 +5617,7 @@ def _gather_pillar(pillarenv, pillar_override, **grains):
# Not sure if these two are correct
__opts__['id'],
__opts__['environment'],
pillar=pillar_override,
pillar_override=pillar_override,
pillarenv=pillarenv
)
ret = pillar.compile_pillar()

4
salt/modules/pillar.py
View file

@ -203,7 +203,7 @@ def items(*args, **kwargs):
__opts__,
__grains__,
__opts__['id'],
pillar=kwargs.get('pillar'),
pillar_override=kwargs.get('pillar'),
pillarenv=kwargs.get('pillarenv') or __opts__['pillarenv'])
return pillar.compile_pillar()
@ -411,7 +411,7 @@ def ext(external, pillar=None):
__opts__['id'],
__opts__['environment'],
ext=external,
pillar=pillar)
pillar_override=pillar)
ret = pillar_obj.compile_pillar()

75
salt/modules/saltutil.py
View file

@ -76,7 +76,8 @@ def _get_top_file_envs():
return __context__['saltutil._top_file_envs']
except KeyError:
try:
st_ = salt.state.HighState(__opts__)
st_ = salt.state.HighState(__opts__,
initial_pillar=__pillar__)
top = st_.get_top()
if top:
envs = list(st_.top_matches(top).keys()) or 'base'
@ -186,10 +187,14 @@ def sync_beacons(saltenv=None, refresh=True):
Sync beacons from ``salt://_beacons`` to the minion
saltenv : base
saltenv
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
If not passed, then all environments configured in the :ref:`top files
<states-top>` will be checked for beacons to sync. If no top files are
found, then the ``base`` environment will be synced.
refresh : True
If ``True``, refresh the available beacons on the minion. This refresh
will be performed even if no new beacons are synced. Set to ``False``
@ -215,10 +220,14 @@ def sync_sdb(saltenv=None):
Sync sdb modules from ``salt://_sdb`` to the minion
saltenv : base
saltenv
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
If not passed, then all environments configured in the :ref:`top files
<states-top>` will be checked for sdb modules to sync. If no top files
are found, then the ``base`` environment will be synced.
refresh : False
This argument has no affect and is included for consistency with the
other sync functions.
@ -241,10 +250,14 @@ def sync_modules(saltenv=None, refresh=True):
Sync execution modules from ``salt://_modules`` to the minion
saltenv : base
saltenv
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
If not passed, then all environments configured in the :ref:`top files
<states-top>` will be checked for execution modules to sync. If no top
files are found, then the ``base`` environment will be synced.
refresh : True
If ``True``, refresh the available execution modules on the minion.
This refresh will be performed even if no new execution modules are
@ -287,10 +300,14 @@ def sync_states(saltenv=None, refresh=True):
Sync state modules from ``salt://_states`` to the minion
saltenv : base
saltenv
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
If not passed, then all environments configured in the :ref:`top files
<states-top>` will be checked for state modules to sync. If no top
files are found, then the ``base`` environment will be synced.
refresh : True
If ``True``, refresh the available states on the minion. This refresh
will be performed even if no new state modules are synced. Set to
@ -349,10 +366,14 @@ def sync_grains(saltenv=None, refresh=True):
Sync grains modules from ``salt://_grains`` to the minion
saltenv : base
saltenv
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
If not passed, then all environments configured in the :ref:`top files
<states-top>` will be checked for grains modules to sync. If no top
files are found, then the ``base`` environment will be synced.
refresh : True
If ``True``, refresh the available execution modules and recompile
pillar data for the minion. This refresh will be performed even if no
@ -380,10 +401,14 @@ def sync_renderers(saltenv=None, refresh=True):
Sync renderers from ``salt://_renderers`` to the minion
saltenv : base
saltenv
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
If not passed, then all environments configured in the :ref:`top files
<states-top>` will be checked for renderers to sync. If no top files
are found, then the ``base`` environment will be synced.
refresh : True
If ``True``, refresh the available execution modules on the minion.
This refresh will be performed even if no new renderers are synced.
@ -410,10 +435,14 @@ def sync_returners(saltenv=None, refresh=True):
Sync beacons from ``salt://_returners`` to the minion
saltenv : base
saltenv
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
If not passed, then all environments configured in the :ref:`top files
<states-top>` will be checked for returners to sync. If no top files
are found, then the ``base`` environment will be synced.
refresh : True
If ``True``, refresh the available execution modules on the minion.
This refresh will be performed even if no new returners are synced. Set
@ -438,10 +467,14 @@ def sync_proxymodules(saltenv=None, refresh=False):
Sync proxy modules from ``salt://_proxy`` to the minion
saltenv : base
saltenv
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
If not passed, then all environments configured in the :ref:`top files
<states-top>` will be checked for proxy modules to sync. If no top
files are found, then the ``base`` environment will be synced.
refresh : True
If ``True``, refresh the available execution modules on the minion.
This refresh will be performed even if no new proxy modules are synced.
@ -467,10 +500,14 @@ def sync_engines(saltenv=None, refresh=False):
Sync engine modules from ``salt://_engines`` to the minion
saltenv : base
saltenv
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
If not passed, then all environments configured in the :ref:`top files
<states-top>` will be checked for engines to sync. If no top files are
found, then the ``base`` environment will be synced.
refresh : True
If ``True``, refresh the available execution modules on the minion.
This refresh will be performed even if no new engine modules are synced.
@ -493,10 +530,14 @@ def sync_output(saltenv=None, refresh=True):
'''
Sync outputters from ``salt://_output`` to the minion
saltenv : base
saltenv
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
If not passed, then all environments configured in the :ref:`top files
<states-top>` will be checked for outputters to sync. If no top files
are found, then the ``base`` environment will be synced.
refresh : True
If ``True``, refresh the available execution modules on the minion.
This refresh will be performed even if no new outputters are synced.
@ -524,10 +565,14 @@ def sync_utils(saltenv=None, refresh=True):
Sync utility modules from ``salt://_utils`` to the minion
saltenv : base
saltenv
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
If not passed, then all environments configured in the :ref:`top files
<states-top>` will be checked for utility modules to sync. If no top
files are found, then the ``base`` environment will be synced.
refresh : True
If ``True``, refresh the available execution modules on the minion.
This refresh will be performed even if no new utility modules are
@ -553,10 +598,14 @@ def sync_log_handlers(saltenv=None, refresh=True):
Sync log handlers from ``salt://_log_handlers`` to the minion
saltenv : base
saltenv
The fileserver environment from which to sync. To sync from more than
one environment, pass a comma-separated list.
If not passed, then all environments configured in the :ref:`top files
<states-top>` will be checked for log handlers to sync. If no top files
are found, then the ``base`` environment will be synced.
refresh : True
If ``True``, refresh the available execution modules on the minion.
This refresh will be performed even if no new log handlers are synced.

312
salt/modules/state.py
View file

@ -252,16 +252,28 @@ def _check_queue(queue, kwargs):
return conflict
def _get_opts(localconfig=None):
def _get_opts(**kwargs):
'''
Return a copy of the opts for use, optionally load a local config on top
'''
opts = copy.deepcopy(__opts__)
if localconfig:
opts = salt.config.minion_config(localconfig, defaults=opts)
if 'localconfig' in kwargs:
opts = salt.config.minion_config(kwargs['localconfig'], defaults=opts)
else:
if 'saltenv' in kwargs:
opts['environment'] = kwargs['saltenv']
if 'pillarenv' in kwargs:
opts['pillarenv'] = kwargs['pillarenv']
return opts
def _get_initial_pillar(opts):
return __pillar__ if __opts__['__cli'] == 'salt-call' \
and opts['pillarenv'] == __opts__['pillarenv'] \
else None
def low(data, queue=False, **kwargs):
'''
Execute a single low data call
@ -325,24 +337,32 @@ def high(data, test=None, queue=False, **kwargs):
conflict = _check_queue(queue, kwargs)
if conflict is not None:
return conflict
opts = _get_opts(kwargs.get('localconfig'))
opts = _get_opts(**kwargs)
opts['test'] = _get_test_value(test, **kwargs)
pillar = kwargs.get('pillar')
pillar_override = kwargs.get('pillar')
pillar_enc = kwargs.get('pillar_enc')
if pillar_enc is None \
and pillar is not None \
and not isinstance(pillar, dict):
and pillar_override is not None \
and not isinstance(pillar_override, dict):
raise SaltInvocationError(
'Pillar data must be formatted as a dictionary, unless pillar_enc '
'is specified.'
)
try:
st_ = salt.state.State(__opts__, pillar, pillar_enc=pillar_enc, proxy=__proxy__,
context=__context__)
st_ = salt.state.State(opts,
pillar_override,
pillar_enc=pillar_enc,
proxy=__proxy__,
context=__context__,
initial_pillar=_get_initial_pillar(opts))
except NameError:
st_ = salt.state.State(__opts__, pillar, pillar_enc=pillar_enc)
st_ = salt.state.State(opts,
pillar_override,
pillar_enc=pillar_enc,
initial_pillar=_get_initial_pillar(opts))
ret = st_.call_high(data)
_set_retcode(ret)
@ -371,15 +391,15 @@ def template(tem, queue=False, **kwargs):
)
kwargs.pop('env')
if 'saltenv' in kwargs:
saltenv = kwargs['saltenv']
else:
saltenv = ''
conflict = _check_queue(queue, kwargs)
if conflict is not None:
return conflict
st_ = salt.state.HighState(__opts__, context=__context__)
opts = _get_opts(**kwargs)
st_ = salt.state.HighState(opts,
context=__context__,
initial_pillar=_get_initial_pillar(opts))
if not _check_pillar(kwargs, st_.opts['pillar']):
__context__['retcode'] = 5
@ -388,7 +408,11 @@ def template(tem, queue=False, **kwargs):
if not tem.endswith('.sls'):
tem = '{sls}.sls'.format(sls=tem)
high_state, errors = st_.render_state(tem, saltenv, '', None, local=True)
high_state, errors = st_.render_state(tem,
kwargs.get('saltenv', ''),
'',
None,
local=True)
if errors:
__context__['retcode'] = 1
return errors
@ -410,10 +434,15 @@ def template_str(tem, queue=False, **kwargs):
conflict = _check_queue(queue, kwargs)
if conflict is not None:
return conflict
opts = _get_opts(**kwargs)
try:
st_ = salt.state.State(__opts__, proxy=__proxy__)
st_ = salt.state.State(opts,
proxy=__proxy__,
initial_pillar=_get_initial_pillar(opts))
except NameError:
st_ = salt.state.State(__opts__)
st_ = salt.state.State(opts, initial_pillar=_get_initial_pillar(opts))
ret = st_.call_template_str(tem)
_set_retcode(ret)
return ret
@ -688,9 +717,7 @@ def run_request(name='default', **kwargs):
return {}
def highstate(test=None,
queue=False,
**kwargs):
def highstate(test=None, queue=False, **kwargs):
'''
Retrieve the state data from the salt master for this minion and execute it
@ -732,7 +759,7 @@ def highstate(test=None,
states to be run with their own custom minion configuration, including
different pillars, file_roots, etc.
mock:
mock
The mock option allows for the state run to execute without actually
calling any states. This then returns a mocked return which will show
the requisite ordering as well as fully validate the state run.
@ -765,7 +792,7 @@ def highstate(test=None,
return conflict
orig_test = __opts__.get('test', None)
opts = _get_opts(kwargs.get('localconfig'))
opts = _get_opts(**kwargs)
opts['test'] = _get_test_value(test, **kwargs)
@ -778,36 +805,32 @@ def highstate(test=None,
)
kwargs.pop('env')
if 'saltenv' in kwargs:
opts['environment'] = kwargs['saltenv']
pillar = kwargs.get('pillar')
pillar_override = kwargs.get('pillar')
pillar_enc = kwargs.get('pillar_enc')
if pillar_enc is None \
and pillar is not None \
and not isinstance(pillar, dict):
and pillar_override is not None \
and not isinstance(pillar_override, dict):
raise SaltInvocationError(
'Pillar data must be formatted as a dictionary, unless pillar_enc '
'is specified.'
)
if 'pillarenv' in kwargs:
opts['pillarenv'] = kwargs['pillarenv']
try:
st_ = salt.state.HighState(opts,
pillar,
pillar_override,
kwargs.get('__pub_jid'),
pillar_enc=pillar_enc,
proxy=__proxy__,
context=__context__,
mocked=kwargs.get('mock', False))
mocked=kwargs.get('mock', False),
initial_pillar=_get_initial_pillar(opts))
except NameError:
st_ = salt.state.HighState(opts,
pillar,
pillar_override,
kwargs.get('__pub_jid'),
pillar_enc=pillar_enc,
mocked=kwargs.get('mock', False))
mocked=kwargs.get('mock', False),
initial_pillar=_get_initial_pillar(opts))
if not _check_pillar(kwargs, st_.opts['pillar']):
__context__['retcode'] = 5
@ -844,13 +867,7 @@ def highstate(test=None,
return ret
def sls(mods,
saltenv=None,
test=None,
exclude=None,
queue=False,
pillarenv=None,
**kwargs):
def sls(mods, test=None, exclude=None, queue=False, **kwargs):
'''
Execute the states in one or more SLS files
@ -894,7 +911,7 @@ def sls(mods,
multiple state runs can safely be run at the same time. Do *not*
use this flag for performance optimization.
saltenv : None
saltenv
Specify a salt fileserver environment to be used when applying states
.. versionchanged:: 0.17.0
@ -946,16 +963,6 @@ def sls(mods,
)
kwargs.pop('env')
if saltenv is None:
if __opts__.get('environment', None):
saltenv = __opts__['environment']
else:
saltenv = 'base'
if not pillarenv:
if __opts__.get('pillarenv', None):
pillarenv = __opts__['pillarenv']
# Modification to __opts__ lost after this if-else
if queue:
_wait(kwargs.get('__pub_jid'))
@ -965,10 +972,6 @@ def sls(mods,
__context__['retcode'] = 1
return conflict
# Ensure desired environment
__opts__['environment'] = saltenv
__opts__['pillarenv'] = pillarenv
if isinstance(mods, list):
disabled = _disabled(mods)
else:
@ -976,20 +979,28 @@ def sls(mods,
if disabled:
for state in disabled:
log.debug('Salt state {0} run is disabled. To re-enable, run state.enable {0}'.format(state))
log.debug(
'Salt state %s is disabled. To re-enable, run '
'state.enable %s', state, state
)
__context__['retcode'] = 1
return disabled
orig_test = __opts__.get('test', None)
opts = _get_opts(kwargs.get('localconfig'))
opts = _get_opts(**kwargs)
opts['test'] = _get_test_value(test, **kwargs)
pillar = kwargs.get('pillar')
# Since this is running a specific SLS file (or files), fall back to the
# 'base' saltenv if none is configured and none was passed.
if opts['environment'] is None:
opts['environment'] = 'base'
pillar_override = kwargs.get('pillar')
pillar_enc = kwargs.get('pillar_enc')
if pillar_enc is None \
and pillar is not None \
and not isinstance(pillar, dict):
and pillar_override is not None \
and not isinstance(pillar_override, dict):
raise SaltInvocationError(
'Pillar data must be formatted as a dictionary, unless pillar_enc '
'is specified.'
@ -1000,20 +1011,23 @@ def sls(mods,
__opts__['cachedir'],
'{0}.cache.p'.format(kwargs.get('cache_name', 'highstate'))
)
try:
st_ = salt.state.HighState(opts,
pillar,
pillar_override,
kwargs.get('__pub_jid'),
pillar_enc=pillar_enc,
proxy=__proxy__,
context=__context__,
mocked=kwargs.get('mock', False))
mocked=kwargs.get('mock', False),
initial_pillar=_get_initial_pillar(opts))
except NameError:
st_ = salt.state.HighState(opts,
pillar,
pillar_override,
kwargs.get('__pub_jid'),
pillar_enc=pillar_enc,
mocked=kwargs.get('mock', False))
mocked=kwargs.get('mock', False),
initial_pillar=_get_initial_pillar(opts))
if not _check_pillar(kwargs, st_.opts['pillar']):
__context__['retcode'] = 5
@ -1036,7 +1050,7 @@ def sls(mods,
st_.push_active()
ret = {}
try:
high_, errors = st_.render_highstate({saltenv: mods})
high_, errors = st_.render_highstate({opts['environment']: mods})
if errors:
__context__['retcode'] = 1
@ -1086,11 +1100,7 @@ def sls(mods,
return ret
def top(topfn,
test=None,
queue=False,
saltenv=None,
**kwargs):
def top(topfn, test=None, queue=False, **kwargs):
'''
Execute a specific top file instead of the default. This is useful to apply
configurations from a different environment (for example, dev or prod), without
@ -1108,20 +1118,24 @@ def top(topfn,
if conflict is not None:
return conflict
orig_test = __opts__.get('test', None)
opts = _get_opts(kwargs.get('localconfig'))
opts = _get_opts(**kwargs)
opts['test'] = _get_test_value(test, **kwargs)
pillar = kwargs.get('pillar')
pillar_override = kwargs.get('pillar')
pillar_enc = kwargs.get('pillar_enc')
if pillar_enc is None \
and pillar is not None \
and not isinstance(pillar, dict):
and pillar_override is not None \
and not isinstance(pillar_override, dict):
raise SaltInvocationError(
'Pillar data must be formatted as a dictionary, unless pillar_enc '
'is specified.'
)
st_ = salt.state.HighState(opts, pillar, pillar_enc=pillar_enc, context=__context__)
st_ = salt.state.HighState(opts,
pillar_override,
pillar_enc=pillar_enc,
context=__context__,
initial_pillar=_get_initial_pillar(opts))
if not _check_pillar(kwargs, st_.opts['pillar']):
__context__['retcode'] = 5
err = ['Pillar failed to render with the following messages:']
@ -1132,8 +1146,8 @@ def top(topfn,
st_.opts['state_top'] = salt.utils.url.create(topfn)
ret = {}
orchestration_jid = kwargs.get('orchestration_jid')
if saltenv:
st_.opts['state_top_saltenv'] = saltenv
if 'saltenv' in kwargs:
st_.opts['state_top_saltenv'] = kwargs['saltenv']
try:
snapper_pre = _snapper_pre(opts, kwargs.get('__pub_jid', 'called localy'))
ret = st_.call_highstate(
@ -1167,17 +1181,22 @@ def show_highstate(queue=False, **kwargs):
conflict = _check_queue(queue, kwargs)
if conflict is not None:
return conflict
pillar = kwargs.get('pillar')
pillar_override = kwargs.get('pillar')
pillar_enc = kwargs.get('pillar_enc')
if pillar_enc is None \
and pillar is not None \
and not isinstance(pillar, dict):
and pillar_override is not None \
and not isinstance(pillar_override, dict):
raise SaltInvocationError(
'Pillar data must be formatted as a dictionary, unless pillar_enc '
'is specified.'
)
st_ = salt.state.HighState(__opts__, pillar, pillar_enc=pillar_enc)
opts = _get_opts(**kwargs)
st_ = salt.state.HighState(opts,
pillar_override,
pillar_enc=pillar_enc,
initial_pillar=_get_initial_pillar(opts))
if not _check_pillar(kwargs, st_.opts['pillar']):
__context__['retcode'] = 5
@ -1207,7 +1226,11 @@ def show_lowstate(queue=False, **kwargs):
if conflict is not None:
assert False
return conflict
st_ = salt.state.HighState(__opts__)
opts = _get_opts(**kwargs)
st_ = salt.state.HighState(opts,
initial_pillar=_get_initial_pillar(opts))
if not _check_pillar(kwargs, st_.opts['pillar']):
__context__['retcode'] = 5
@ -1222,13 +1245,7 @@ def show_lowstate(queue=False, **kwargs):
return ret
def sls_id(
id_,
mods,
saltenv='base',
test=None,
queue=False,
**kwargs):
def sls_id(id_, mods, test=None, queue=False, **kwargs):
'''
Call a single ID from the named module(s) and handle all requisites
@ -1246,14 +1263,21 @@ def sls_id(
if conflict is not None:
return conflict
orig_test = __opts__.get('test', None)
opts = _get_opts(kwargs.get('localconfig'))
opts = _get_opts(**kwargs)
opts['test'] = _get_test_value(test, **kwargs)
if 'pillarenv' in kwargs:
opts['pillarenv'] = kwargs['pillarenv']
# Since this is running a specific ID within a specific SLS file, fall back
# to the 'base' saltenv if none is configured and none was passed.
if opts['environment'] is None:
opts['environment'] = 'base'
try:
st_ = salt.state.HighState(opts, proxy=__proxy__)
st_ = salt.state.HighState(opts,
proxy=__proxy__,
initial_pillar=_get_initial_pillar(opts))
except NameError:
st_ = salt.state.HighState(opts)
st_ = salt.state.HighState(opts,
initial_pillar=_get_initial_pillar(opts))
if not _check_pillar(kwargs, st_.opts['pillar']):
__context__['retcode'] = 5
@ -1265,7 +1289,7 @@ def sls_id(
split_mods = mods.split(',')
st_.push_active()
try:
high_, errors = st_.render_highstate({saltenv: split_mods})
high_, errors = st_.render_highstate({opts['environment']: split_mods})
finally:
st_.pop_active()
errors += st_.state.verify_high(high_)
@ -1285,16 +1309,12 @@ def sls_id(
if not ret:
raise SaltInvocationError(
'No matches for ID \'{0}\' found in SLS \'{1}\' within saltenv '
'\'{2}\''.format(id_, mods, saltenv)
'\'{2}\''.format(id_, mods, opts['environment'])
)
return ret
def show_low_sls(mods,
saltenv='base',
test=None,
queue=False,
**kwargs):
def show_low_sls(mods, test=None, queue=False, **kwargs):
'''
Display the low data from a specific sls. The default environment is
``base``, use ``saltenv`` to specify a different environment.
@ -1304,6 +1324,7 @@ def show_low_sls(mods,
.. code-block:: bash
salt '*' state.show_low_sls foo
salt '*' state.show_low_sls foo saltenv=dev
'''
if 'env' in kwargs:
salt.utils.warn_until(
@ -1318,11 +1339,15 @@ def show_low_sls(mods,
if conflict is not None:
return conflict
orig_test = __opts__.get('test', None)
opts = _get_opts(kwargs.get('localconfig'))
opts = _get_opts(**kwargs)
opts['test'] = _get_test_value(test, **kwargs)
if 'pillarenv' in kwargs:
opts['pillarenv'] = kwargs['pillarenv']
st_ = salt.state.HighState(opts)
# Since this is dealing with a specific SLS file (or files), fall back to
# the 'base' saltenv if none is configured and none was passed.
if opts['environment'] is None:
opts['environment'] = 'base'
st_ = salt.state.HighState(opts, initial_pillar=_get_initial_pillar(opts))
if not _check_pillar(kwargs, st_.opts['pillar']):
__context__['retcode'] = 5
@ -1333,7 +1358,7 @@ def show_low_sls(mods,
mods = mods.split(',')
st_.push_active()
try:
high_, errors = st_.render_highstate({saltenv: mods})
high_, errors = st_.render_highstate({opts['environment']: mods})
finally:
st_.pop_active()
errors += st_.state.verify_high(high_)
@ -1347,7 +1372,7 @@ def show_low_sls(mods,
return ret
def show_sls(mods, saltenv='base', test=None, queue=False, **kwargs):
def show_sls(mods, test=None, queue=False, **kwargs):
'''
Display the state data from a specific sls or list of sls files on the
master. The default environment is ``base``, use ``saltenv`` to specify a
@ -1377,24 +1402,29 @@ def show_sls(mods, saltenv='base', test=None, queue=False, **kwargs):
if conflict is not None:
return conflict
orig_test = __opts__.get('test', None)
opts = _get_opts(kwargs.get('localconfig'))
opts = _get_opts(**kwargs)
opts['test'] = _get_test_value(test, **kwargs)
pillar = kwargs.get('pillar')
# Since this is dealing with a specific SLS file (or files), fall back to
# the 'base' saltenv if none is configured and none was passed.
if opts['environment'] is None:
opts['environment'] = 'base'
pillar_override = kwargs.get('pillar')
pillar_enc = kwargs.get('pillar_enc')
if pillar_enc is None \
and pillar is not None \
and not isinstance(pillar, dict):
and pillar_override is not None \
and not isinstance(pillar_override, dict):
raise SaltInvocationError(
'Pillar data must be formatted as a dictionary, unless pillar_enc '
'is specified.'
)
if 'pillarenv' in kwargs:
opts['pillarenv'] = kwargs['pillarenv']
st_ = salt.state.HighState(opts, pillar, pillar_enc=pillar_enc)
st_ = salt.state.HighState(opts,
pillar_override,
pillar_enc=pillar_enc,
initial_pillar=_get_initial_pillar(opts))
if not _check_pillar(kwargs, st_.opts['pillar']):
__context__['retcode'] = 5
@ -1405,7 +1435,7 @@ def show_sls(mods, saltenv='base', test=None, queue=False, **kwargs):
mods = mods.split(',')
st_.push_active()
try:
high_, errors = st_.render_highstate({saltenv: mods})
high_, errors = st_.render_highstate({opts['environment']: mods})
finally:
st_.pop_active()
errors += st_.state.verify_high(high_)
@ -1428,8 +1458,6 @@ def show_top(queue=False, **kwargs):
salt '*' state.show_top
'''
opts = copy.deepcopy(__opts__)
if 'env' in kwargs:
salt.utils.warn_until(
'Oxygen',
@ -1439,12 +1467,13 @@ def show_top(queue=False, **kwargs):
)
kwargs.pop('env')
if 'saltenv' in kwargs:
opts['environment'] = kwargs['saltenv']
conflict = _check_queue(queue, kwargs)
if conflict is not None:
return conflict
st_ = salt.state.HighState(opts)
opts = _get_opts(**kwargs)
st_ = salt.state.HighState(opts, initial_pillar=_get_initial_pillar(opts))
if not _check_pillar(kwargs, st_.opts['pillar']):
__context__['retcode'] = 5
@ -1490,23 +1519,30 @@ def single(fun, name, test=None, queue=False, **kwargs):
'__id__': name,
'name': name})
orig_test = __opts__.get('test', None)
opts = _get_opts(kwargs.get('localconfig'))
opts = _get_opts(**kwargs)
opts['test'] = _get_test_value(test, **kwargs)
pillar = kwargs.get('pillar')
pillar_override = kwargs.get('pillar')
pillar_enc = kwargs.get('pillar_enc')
if pillar_enc is None \
and pillar is not None \
and not isinstance(pillar, dict):
and pillar_override is not None \
and not isinstance(pillar_override, dict):
raise SaltInvocationError(
'Pillar data must be formatted as a dictionary, unless pillar_enc '
'is specified.'
)
try:
st_ = salt.state.State(opts, pillar, pillar_enc=pillar_enc, proxy=__proxy__)
st_ = salt.state.State(opts,
pillar_override,
pillar_enc=pillar_enc,
proxy=__proxy__,
initial_pillar=_get_initial_pillar(opts))
except NameError:
st_ = salt.state.State(opts, pillar, pillar_enc=pillar_enc)
st_ = salt.state.State(opts,
pillar_override,
pillar_enc=pillar_enc,
initial_pillar=_get_initial_pillar(opts))
err = st_.verify_data(kwargs)
if err:
__context__['retcode'] = 1
@ -1549,7 +1585,11 @@ def clear_cache():
return ret
def pkg(pkg_path, pkg_sum, hash_type, test=None, **kwargs):
def pkg(pkg_path,
pkg_sum,
hash_type,
test=None,
**kwargs):
'''
Execute a packaged state run, the packaged state run will exist in a
tarball available locally. This packaged state
@ -1587,10 +1627,10 @@ def pkg(pkg_path, pkg_sum, hash_type, test=None, **kwargs):
pillar_json = os.path.join(root, 'pillar.json')
if os.path.isfile(pillar_json):
with salt.utils.fopen(pillar_json, 'r') as fp_:
pillar = json.load(fp_)
pillar_override = json.load(fp_)
else:
pillar = None
popts = _get_opts(kwargs.get('localconfig'))
pillar_override = None
popts = _get_opts(**kwargs)
popts['fileclient'] = 'local'
popts['file_roots'] = {}
popts['test'] = _get_test_value(test, **kwargs)
@ -1600,7 +1640,7 @@ def pkg(pkg_path, pkg_sum, hash_type, test=None, **kwargs):
if not os.path.isdir(full):
continue
popts['file_roots'][fn_] = [full]
st_ = salt.state.State(popts, pillar=pillar)
st_ = salt.state.State(popts, pillar_override=pillar_override)
snapper_pre = _snapper_pre(popts, kwargs.get('__pub_jid', 'called localy'))
ret = st_.call_chunks(lowstate)
try:

72
salt/pillar/__init__.py
View file

@ -32,7 +32,7 @@ log = logging.getLogger(__name__)
def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None,
pillar=None, pillarenv=None):
pillar_override=None, pillarenv=None):
'''
Return the correct pillar driver based on the file_client option
'''
@ -49,14 +49,14 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None,
log.info('Compiling pillar from cache')
log.debug('get_pillar using pillar cache with ext: {0}'.format(ext))
return PillarCache(opts, grains, minion_id, saltenv, ext=ext, functions=funcs,
pillar=pillar, pillarenv=pillarenv)
pillar_override=pillar_override, pillarenv=pillarenv)
return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs,
pillar=pillar, pillarenv=pillarenv)
pillar_override=pillar_override, pillarenv=pillarenv)
# TODO: migrate everyone to this one!
def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None,
pillar=None, pillarenv=None):
pillar_override=None, pillarenv=None):
'''
Return the correct pillar driver based on the file_client option
'''
@ -68,7 +68,7 @@ def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None
'local': AsyncPillar,
}.get(file_client, AsyncPillar)
return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs,
pillar=pillar, pillarenv=pillarenv)
pillar_override=pillar_override, pillarenv=pillarenv)
class AsyncRemotePillar(object):
@ -76,7 +76,7 @@ class AsyncRemotePillar(object):
Get the pillar from the master
'''
def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None,
pillar=None, pillarenv=None):
pillar_override=None, pillarenv=None):
self.opts = opts
self.opts['environment'] = saltenv
self.ext = ext
@ -85,12 +85,10 @@ class AsyncRemotePillar(object):
self.channel = salt.transport.client.AsyncReqChannel.factory(opts)
if pillarenv is not None or 'pillarenv' not in self.opts:
self.opts['pillarenv'] = pillarenv
self.pillar_override = {}
if pillar is not None:
if isinstance(pillar, dict):
self.pillar_override = pillar
else:
log.error('Pillar data must be a dictionary')
self.pillar_override = pillar_override or {}
if not isinstance(self.pillar_override, dict):
self.pillar_override = {}
log.error('Pillar data must be a dictionary')
@tornado.gen.coroutine
def compile_pillar(self):
@ -129,7 +127,7 @@ class RemotePillar(object):
Get the pillar from the master
'''
def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None,
pillar=None, pillarenv=None):
pillar_override=None, pillarenv=None):
self.opts = opts
self.opts['environment'] = saltenv
self.ext = ext
@ -138,12 +136,10 @@ class RemotePillar(object):
self.channel = salt.transport.Channel.factory(opts)
if pillarenv is not None or 'pillarenv' not in self.opts:
self.opts['pillarenv'] = pillarenv
self.pillar_override = {}
if pillar is not None:
if isinstance(pillar, dict):
self.pillar_override = pillar
else:
log.error('Pillar data must be a dictionary')
self.pillar_override = pillar_override or {}
if not isinstance(self.pillar_override, dict):
self.pillar_override = {}
log.error('Pillar data must be a dictionary')
def compile_pillar(self):
'''
@ -175,8 +171,8 @@ class PillarCache(object):
'''
Return a cached pillar if it exists, otherwise cache it.
Pillar caches are structed in two diminensions: minion_id with a dict of saltenvs.
Each saltenv contains a pillar dict
Pillar caches are structed in two diminensions: minion_id with a dict of
saltenvs. Each saltenv contains a pillar dict
Example data structure:
@ -187,7 +183,7 @@ class PillarCache(object):
'''
# TODO ABC?
def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None,
pillar=None, pillarenv=None):
pillar_override=None, pillarenv=None):
# Yes, we need all of these because we need to route to the Pillar object
# if we have no cache. This is another refactor target.
@ -197,7 +193,7 @@ class PillarCache(object):
self.minion_id = minion_id
self.ext = ext
self.functions = functions
self.pillar = pillar
self.pillar_override = pillar_override
self.pillarenv = pillarenv
if saltenv is None:
@ -226,13 +222,13 @@ class PillarCache(object):
'''
log.debug('Pillar cache getting external pillar with ext: {0}'.format(self.ext))
fresh_pillar = Pillar(self.opts,
self.grains,
self.minion_id,
self.saltenv,
ext=self.ext,
functions=self.functions,
pillar=self.pillar,
pillarenv=self.pillarenv)
self.grains,
self.minion_id,
self.saltenv,
ext=self.ext,
functions=self.functions,
pillar_override=self.pillar_override,
pillarenv=self.pillarenv)
return fresh_pillar.compile_pillar() # FIXME We are not yet passing pillar_dirs in here
def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs
@ -265,7 +261,7 @@ class Pillar(object):
Read over the pillar top files and render the pillar data
'''
def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None,
pillar=None, pillarenv=None):
pillar_override=None, pillarenv=None):
self.minion_id = minion_id
self.ext = ext
# Store the file_roots path so we can restore later. Issue 5449
@ -303,12 +299,10 @@ class Pillar(object):
self.ext_pillars = salt.loader.pillars(ext_pillar_opts, self.functions)
self.ignored_pillars = {}
self.pillar_override = {}
if pillar is not None:
if isinstance(pillar, dict):
self.pillar_override = pillar
else:
log.error('Pillar data must be a dictionary')
self.pillar_override = pillar_override or {}
if not isinstance(self.pillar_override, dict):
self.pillar_override = {}
log.error('Pillar data must be a dictionary')
def __valid_on_demand_ext_pillar(self, opts):
'''
@ -784,7 +778,7 @@ class Pillar(object):
return pillar, errors
ext = None
# Bring in CLI pillar data
if self.pillar_override and isinstance(self.pillar_override, dict):
if self.pillar_override:
pillar = merge(pillar,
self.pillar_override,
self.merge_strategy,
@ -863,7 +857,7 @@ class Pillar(object):
log.critical('Pillar render error: {0}'.format(error))
pillar['_errors'] = errors
if self.pillar_override and isinstance(self.pillar_override, dict):
if self.pillar_override:
pillar = merge(pillar,
self.pillar_override,
self.merge_strategy,

2
salt/runners/state.py
View file

@ -57,10 +57,10 @@ def orchestrate(mods,
minion = salt.minion.MasterMinion(__opts__)
running = minion.functions['state.sls'](
mods,
saltenv,
test,
exclude,
pillar=pillar,
saltenv=saltenv,
pillarenv=pillarenv,
orchestration_jid=orchestration_jid)
ret = {'data': {minion.opts['id']: running}, 'outputter': 'highstate'}

25
salt/state.py
View file

@ -635,19 +635,20 @@ class State(object):
def __init__(
self,
opts,
pillar=None,
pillar_override=None,
jid=None,
pillar_enc=None,
proxy=None,
context=None,
mocked=False,
loader='states'):
loader='states',
initial_pillar=None):
self.states_loader = loader
if 'grains' not in opts:
opts['grains'] = salt.loader.grains(opts)
self.opts = opts
self.proxy = proxy
self._pillar_override = pillar
self._pillar_override = pillar_override
if pillar_enc is not None:
try:
pillar_enc = pillar_enc.lower()
@ -659,7 +660,8 @@ class State(object):
.format(', '.join(VALID_PILLAR_ENC))
)
self._pillar_enc = pillar_enc
self.opts['pillar'] = self._gather_pillar()
self.opts['pillar'] = initial_pillar if initial_pillar is not None \
else self._gather_pillar()
self.state_con = context or {}
self.load_modules()
self.active = set()
@ -698,7 +700,7 @@ class State(object):
self.opts['grains'],
self.opts['id'],
self.opts['environment'],
pillar=self._pillar_override,
pillar_override=self._pillar_override,
pillarenv=self.opts.get('pillarenv')
)
ret = pillar.compile_pillar()
@ -3514,25 +3516,26 @@ class HighState(BaseHighState):
def __init__(
self,
opts,
pillar=None,
pillar_override=None,
jid=None,
pillar_enc=None,
proxy=None,
context=None,
mocked=False,
loader='states'):
loader='states',
initial_pillar=None):
self.opts = opts
self.client = salt.fileclient.get_file_client(self.opts)
BaseHighState.__init__(self, opts)
self.state = State(
self.opts,
pillar,
self.state = State(self.opts,
pillar_override,
jid,
pillar_enc,
proxy=proxy,
context=context,
mocked=mocked,
loader=loader)
loader=loader,
initial_pillar=initial_pillar)
self.matcher = salt.minion.Matcher(self.opts)
self.proxy = proxy

38
tests/unit/modules/state_test.py
View file

@ -28,7 +28,9 @@ from salt.modules import state
# Globals
state.__salt__ = {}
state.__context__ = {}
state.__opts__ = {'cachedir': '/D'}
state.__opts__ = {'cachedir': '/D',
'environment': None,
'__cli': 'salt'}
state.__pillar__ = {}
@ -45,7 +47,11 @@ class MockState(object):
'''
flag = None
def __init__(self, opts, pillar=False, pillar_enc=None):
def __init__(self,
opts,
pillar_override=False,
pillar_enc=None,
initial_pillar=None):
pass
def verify_data(self, data):
@ -134,9 +140,9 @@ class MockState(object):
opts = {'state_top': '',
'pillar': {}}
def __init__(self, opts, pillar=None, *args, **kwargs):
def __init__(self, opts, pillar_override=None, *args, **kwargs):
self.state = MockState.State(opts,
pillar=pillar)
pillar_override=pillar_override)
def render_state(self, sls, saltenv, mods, matches, local=False):
'''
@ -576,7 +582,10 @@ class StateTestCase(TestCase):
self.assertEqual(state.sls_id("apache", "http"), "A")
with patch.dict(state.__opts__, {"test": "A"}):
mock = MagicMock(return_value={'test': True})
mock = MagicMock(
return_value={'test': True,
'environment': None}
)
with patch.object(state, '_get_opts', mock):
mock = MagicMock(return_value=True)
with patch.object(salt.utils, 'test_mode', mock):
@ -600,7 +609,10 @@ class StateTestCase(TestCase):
self.assertEqual(state.show_low_sls("foo"), "A")
with patch.dict(state.__opts__, {"test": "A"}):
mock = MagicMock(return_value={'test': True})
mock = MagicMock(
return_value={'test': True,
'environment': None}
)
with patch.object(state, '_get_opts', mock):
MockState.State.flag = True
MockState.HighState.flag = True
@ -619,7 +631,10 @@ class StateTestCase(TestCase):
self.assertEqual(state.show_sls("foo"), "A")
with patch.dict(state.__opts__, {"test": "A"}):
mock = MagicMock(return_value={'test': True})
mock = MagicMock(
return_value={'test': True,
'environment': None}
)
with patch.object(state, '_get_opts', mock):
mock = MagicMock(return_value=True)
with patch.object(salt.utils, 'test_mode', mock):
@ -798,7 +813,6 @@ class StateTestCase(TestCase):
with patch.dict(state.__context__, {"retcode": 1}):
self.assertEqual(
state.sls("core,edit.vim dev",
None,
None,
None,
True),
@ -813,13 +827,13 @@ class StateTestCase(TestCase):
with patch.dict(state.__context__, {"retcode": 5}):
with patch.dict(state.__pillar__, {"_errors": "E1"}):
self.assertListEqual(state.sls("core,edit.vim dev",
None,
None,
None,
True), ret)
with patch.dict(state.__opts__, {"test": None}):
mock = MagicMock(return_value={"test": ""})
mock = MagicMock(return_value={"test": "",
"environment": None})
with patch.object(state, '_get_opts', mock):
mock = MagicMock(return_value=True)
with patch.object(salt.utils,
@ -831,7 +845,6 @@ class StateTestCase(TestCase):
"core,edit.vim dev",
None,
None,
None,
True,
pillar="A")
@ -848,7 +861,6 @@ class StateTestCase(TestCase):
mock_open()):
self.assertTrue(
state.sls(arg,
None,
None,
None,
True,
@ -862,7 +874,6 @@ class StateTestCase(TestCase):
".vim dev",
None,
None,
None,
True)
)
@ -899,7 +910,6 @@ class StateTestCase(TestCase):
".vim dev",
None,
None,
None,
True))
@patch('salt.modules.state.tarfile', MockTarFile)