Merge branch '2016.11' into 'develop'

Conflicts:
  - salt/pillar/__init__.py
This commit is contained in:
rallytime 2017-01-30 09:06:27 -07:00
commit 8928436bdd
22 changed files with 323 additions and 78 deletions

View file

@ -700,6 +700,11 @@
# ext_pillar.
#ext_pillar_first: False
# The external pillars permitted to be used on-demand using pillar.ext
#on_demand_ext_pillar:
# - libvirt
# - virtkey
# The pillar_gitfs_ssl_verify option specifies whether to ignore ssl certificate
# errors when contacting the pillar gitfs backend. You might want to set this to
# false if you're using a git backend that uses a self-signed certificate but

View file

@ -2548,6 +2548,34 @@ configuration is the same as :conf_master:`file_roots`:
prod:
- /srv/pillar/prod
.. conf_master:: on_demand_ext_pillar
``on_demand_ext_pillar``
------------------------
.. versionadded:: 2016.3.6,2016.11.3,Nitrogen
Default: ``['libvirt', 'virtkey']``
The external pillars permitted to be used on-demand using :py:func:`pillar.ext
<salt.modules.pillar.ext>`.
.. code-block:: yaml
on_demand_ext_pillar:
- libvirt
- virtkey
- git
.. warning::
This will allow minions to request specific pillar data via
:py:func:`pillar.ext <salt.modules.pillar.ext>`, and may be considered a
security risk. However, pillar data generated in this way will not affect
the :ref:`in-memory pillar data <pillar-in-memory>`, so this risk is
limited to instances in which states/modules/etc. (built-in or custom) rely
upon pillar data generated by :py:func:`pillar.ext
<salt.modules.pillar.ext>`.
.. conf_master:: pillar_opts
``pillar_opts``

View file

@ -1608,6 +1608,35 @@ the pillar environments.
prod:
- /srv/pillar/prod
.. conf_minion:: on_demand_ext_pillar
``on_demand_ext_pillar``
------------------------
.. versionadded:: 2016.3.6,2016.11.3,Nitrogen
Default: ``['libvirt', 'virtkey']``
When using a local :conf_minion:`file_client`, this option controls which
external pillars are permitted to be used on-demand using :py:func:`pillar.ext
<salt.modules.pillar.ext>`.
.. code-block:: yaml
on_demand_ext_pillar:
- libvirt
- virtkey
- git
.. warning::
This will allow a masterless minion to request specific pillar data via
:py:func:`pillar.ext <salt.modules.pillar.ext>`, and may be considered a
security risk. However, pillar data generated in this way will not affect
the :ref:`in-memory pillar data <pillar-in-memory>`, so this risk is
limited to instances in which states/modules/etc. (built-in or custom) rely
upon pillar data generated by :py:func:`pillar.ext
<salt.modules.pillar.ext>`.
.. conf_minion:: pillarenv
``pillarenv``

View file

@ -96,9 +96,9 @@ variable in a Salt state.
.. code-block:: yaml
Create a file with contents from an environment variable:
file.managed:
- name: /tmp/hello
- contents: {{ salt['environ.get']('MYENVVAR') }}
file.managed:
- name: /tmp/hello
- contents: {{ salt['environ.get']('MYENVVAR') }}
Error checking:
@ -115,8 +115,7 @@ Error checking:
{% else %}
Fail - no environment passed in:
test:
A. fail_without_changes
test.fail_without_changes
{% endif %}

View file

@ -11,7 +11,7 @@ GitPython==1.0.1
idna==2.0
ioflo==1.5.0
ipaddress==1.0.16
Jinja2==2.8
Jinja2==2.9.4
libnacl==1.4.4
linode-python==1.1.1
Mako==1.0.3

70
pkg/smartos/salt-api.xml Normal file
View file

@ -0,0 +1,70 @@
<?xml version="1.0"?>
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<!--
Created by Manifold
-->
<service_bundle type="manifest" name="salt-api">
<service name="network/salt-api" type="service" version="1">
<create_default_instance enabled="false"/>
<single_instance/>
<dependency name="config-file"
grouping="require_all"
restart_on="none"
type="path">
<service_fmri value='file:///opt/local/etc/salt/minion'/>
</dependency>
<dependency name="network"
grouping="require_all"
restart_on="error"
type="service">
<service_fmri value="svc:/milestone/network:default"/>
</dependency>
<dependency name="filesystem"
grouping="require_all"
restart_on="error"
type="service">
<service_fmri value="svc:/system/filesystem/local"/>
</dependency>
<method_context/>
<exec_method type="method"
name="start"
exec="/opt/local/bin/salt-api -c %{config_dir}"
timeout_seconds="60"/>
<exec_method type="method"
name="stop"
exec=":kill"
timeout_seconds="60"/>
<property_group name="startd" type="framework">
<propval name="duration" type="astring" value="child"/>
<propval name="ignore_error" type="astring" value="core,signal"/>
</property_group>
<property_group name="application" type="application">
<propval name="config_file" type="astring" value="/opt/local/etc/salt/master"/>
<propval name="config_dir" type="astring" value="/opt/local/etc/salt"/>
</property_group>
<stability value="Unstable"/>
<template>
<common_name>
<loctext xml:lang="C">Salt API</loctext>
</common_name>
<documentation>
<doc_link name="SaltStack Documentation"
uri="http://docs.saltstack.com"/>
</documentation>
</template>
</service>
</service_bundle>

View file

@ -25,6 +25,14 @@ authenticated against. This defaults to `login`
The Python interface to PAM does not support authenticating as ``root``.
.. note:: Using PAM groups with SSSD groups on python2.
To use sssd with the PAM eauth module and groups the `pysss` module is
needed. On RedHat/CentOS this is `python-sss`.
This should not be needed with python >= 3.3, because the `os` modules has the
`getgrouplist` function.
'''
# Import Python Libs

View file

@ -259,6 +259,9 @@ VALID_OPTS = {
# A map of saltenvs and fileserver backend locations
'pillar_roots': dict,
# The external pillars permitted to be used on-demand using pillar.ext
'on_demand_ext_pillar': list,
# The type of hashing algorithm to use when doing file comparisons
'hash_type': str,
@ -1041,6 +1044,7 @@ DEFAULT_MINION_OPTS = {
'base': [salt.syspaths.BASE_PILLAR_ROOTS_DIR,
salt.syspaths.SPM_PILLAR_PATH]
},
'on_demand_ext_pillar': ['libvirt', 'virtkey'],
'git_pillar_base': 'master',
'git_pillar_branch': 'master',
'git_pillar_env': '',
@ -1241,6 +1245,7 @@ DEFAULT_MASTER_OPTS = {
'base': [salt.syspaths.BASE_PILLAR_ROOTS_DIR,
salt.syspaths.SPM_PILLAR_PATH]
},
'on_demand_ext_pillar': ['libvirt', 'virtkey'],
'thorium_interval': 0.5,
'thorium_roots': {
'base': [salt.syspaths.BASE_THORIUM_ROOTS_DIR],

View file

@ -19,7 +19,6 @@ them onto a logstash endpoint.
# Import python libraries
from __future__ import absolute_import
import logging
import json
# Import salt libs
import salt.utils.event
@ -66,4 +65,4 @@ def start(host, port=5959, tag='salt/engine/logstash'):
while True:
event = event_bus.get_event()
if event:
logstash_logger.info(tag, extra=json.dumps(event))
logstash_logger.info(tag, extra=event)

View file

@ -1882,9 +1882,9 @@ class ClearFuncs(object):
'user': username}
self.event.fire_event(data, tagify([jid, 'new'], 'wheel'))
ret = self.wheel_.call_func(fun, **clear_load)
data['return'] = ret
data['success'] = True
ret = self.wheel_.call_func(fun, full_return=True, **clear_load)
data['return'] = ret['return']
data['success'] = ret['success']
self.event.fire_event(data, tagify([jid, 'ret'], 'wheel'))
return {'tag': tag,
'data': data}

View file

@ -181,7 +181,7 @@ def build(format='qcow2', path='/tmp/'):
CLI Example:
.. code-block:: bash:
.. code-block:: bash
salt myminion inspector.build
salt myminion inspector.build format=iso path=/opt/builds/
@ -209,7 +209,7 @@ def export(local=False, path="/tmp", format='qcow2'):
CLI Example:
.. code-block:: bash:
.. code-block:: bash
salt myminion inspector.export
salt myminion inspector.export format=iso path=/opt/builds/
@ -232,7 +232,7 @@ def snapshots():
CLI Example:
.. code-block:: bash:
.. code-block:: bash
salt myminion inspector.snapshots
'''
@ -254,7 +254,7 @@ def delete(all=False, *databases):
CLI example:
.. code-block:: bash:
.. code-block:: bash
salt myminion inspector.delete <ID> <ID1> <ID2>..
salt myminion inspector.delete all=True

View file

@ -103,11 +103,17 @@ def _available_services():
plist = plistlib.readPlistFromBytes(
salt.utils.to_bytes(plist_xml))
available_services[plist.Label.lower()] = {
'filename': filename,
'file_path': true_path,
'plist': plist,
}
try:
available_services[plist.Label.lower()] = {
'filename': filename,
'file_path': true_path,
'plist': plist,
}
except AttributeError:
# As of MacOS 10.12 there might be plist files without Label key
# in the searched directories. As these files do not represent
# services, thay are not added to the list.
pass
return available_services

View file

@ -252,15 +252,8 @@ def item(*args, **kwargs):
'''
.. versionadded:: 0.16.2
Return one or more pillar entries
pillar
If specified, allows for a dictionary of pillar data to be made
available to pillar and ext_pillar rendering. these pillar variables
will also override any variables of the same name in pillar or
ext_pillar.
.. versionadded:: 2015.5.0
Return one or more pillar entries from the :ref:`in-memory pillar data
<pillar-in-memory>`.
delimiter
Delimiter used to traverse nested dictionaries.
@ -288,14 +281,14 @@ def item(*args, **kwargs):
'''
ret = {}
default = kwargs.get('default', '')
delimiter = kwargs.get('delimiter', ':')
delimiter = kwargs.get('delimiter', DEFAULT_TARGET_DELIM)
try:
for arg in args:
ret[arg] = salt.utils.traverse_dict_and_list(__pillar__,
arg,
default,
delimiter)
arg,
default,
delimiter)
except KeyError:
pass
@ -330,9 +323,39 @@ def raw(key=None):
def ext(external, pillar=None):
'''
.. versionchanged:: 2016.3.6,2016.11.3,Nitrogen
The supported ext_pillar types are now tunable using the
:conf_master:`on_demand_ext_pillar` config option. Earlier releases
used a hard-coded default.
Generate the pillar and apply an explicit external pillar
CLI Example:
external
A single ext_pillar to add to the ext_pillar configuration. This must
be passed as a single section from the ext_pillar configuration (see
CLI examples below). For more complicated ``ext_pillar``
configurations, it can be helpful to use the Python shell to load YAML
configuration into a dictionary, and figure out
.. code-block:: python
>>> import yaml
>>> ext_pillar = yaml.safe_load("""
... ext_pillar:
... - git:
... - issue38440 https://github.com/terminalmage/git_pillar:
... - env: base
... """)
>>> ext_pillar
{'ext_pillar': [{'git': [{'mybranch https://github.com/myuser/myrepo': [{'env': 'base'}]}]}]}
>>> ext_pillar['ext_pillar'][0]
{'git': [{'mybranch https://github.com/myuser/myrepo': [{'env': 'base'}]}]}
In the above example, the value to pass would be
``{'git': [{'mybranch https://github.com/myuser/myrepo': [{'env': 'base'}]}]}``.
Note that this would need to be quoted when passing on the CLI (as in
the CLI examples below).
pillar : None
If specified, allows for a dictionary of pillar data to be made
@ -342,9 +365,13 @@ def ext(external, pillar=None):
.. versionadded:: 2015.5.0
CLI Examples:
.. code-block:: bash
salt '*' pillar.ext '{libvirt: _}'
salt '*' pillar.ext "{'git': ['master https://github.com/myuser/myrepo']}"
salt '*' pillar.ext "{'git': [{'mybranch https://github.com/myuser/myrepo': [{'env': 'base'}]}]}"
'''
if isinstance(external, six.string_types):
external = yaml.safe_load(external)

View file

@ -270,13 +270,14 @@ class Pillar(object):
def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None,
pillar=None, pillarenv=None, rend=None):
self.minion_id = minion_id
self.ext = ext
if pillarenv is None:
if opts.get('pillarenv_from_saltenv', False):
opts['pillarenv'] = saltenv
# Store the file_roots path so we can restore later. Issue 5449
self.actual_file_roots = opts['file_roots']
# use the local file client
self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, ext=ext, pillarenv=pillarenv)
self.opts = self.__gen_opts(opts, grains, saltenv=saltenv, pillarenv=pillarenv)
self.saltenv = saltenv
self.client = salt.fileclient.get_file_client(self.opts, True)
@ -318,16 +319,39 @@ class Pillar(object):
else:
log.error('Pillar data must be a dictionary')
def __valid_ext(self, ext):
def __valid_on_demand_ext_pillar(self, opts):
'''
Check to see if the on demand external pillar is allowed
'''
if not isinstance(ext, dict):
return {}
valid = set(('libvirt', 'virtkey'))
if any(key not in valid for key in ext):
return {}
return ext
if not isinstance(self.ext, dict):
log.error(
'On-demand pillar %s is not formatted as a dictionary',
self.ext
)
return False
on_demand = opts.get('on_demand_ext_pillar', [])
try:
invalid_on_demand = set([x for x in self.ext if x not in on_demand])
except TypeError:
# Prevent traceback when on_demand_ext_pillar option is malformed
log.error(
'The \'on_demand_ext_pillar\' configuration option is '
'malformed, it should be a list of ext_pillar module names'
)
return False
if invalid_on_demand:
log.error(
'The following ext_pillar modules are not allowed for '
'on-demand pillar data: %s. Valid on-demand ext_pillar '
'modules are: %s. The valid modules can be adjusted by '
'setting the \'on_demand_ext_pillar\' config option.',
', '.join(sorted(invalid_on_demand)),
', '.join(on_demand),
)
return False
return True
def __gen_opts(self, opts_in, grains, saltenv=None, ext=None, pillarenv=None):
'''
@ -351,11 +375,11 @@ class Pillar(object):
opts['state_top'] = salt.utils.url.create(opts['state_top'][1:])
else:
opts['state_top'] = salt.utils.url.create(opts['state_top'])
if self.__valid_ext(ext):
if self.ext and self.__valid_on_demand_ext_pillar(opts):
if 'ext_pillar' in opts:
opts['ext_pillar'].append(ext)
opts['ext_pillar'].append(self.ext)
else:
opts['ext_pillar'] = [ext]
opts['ext_pillar'] = [self.ext]
return opts
def _get_envs(self):
@ -742,6 +766,21 @@ class Pillar(object):
'''
if errors is None:
errors = []
try:
# Make sure that on-demand git_pillar is fetched before we try to
# compile the pillar data. git_pillar will fetch a remote when
# the git ext_pillar() func is run, but only for masterless.
if self.ext and 'git' in self.ext \
and self.opts.get('__role') != 'minion':
# Avoid circular import
import salt.utils.gitfs
from salt.pillar.git_pillar import PER_REMOTE_OVERRIDES
git_pillar = salt.utils.gitfs.GitPillar(self.opts)
git_pillar.init_remotes(self.ext['git'], PER_REMOTE_OVERRIDES)
git_pillar.fetch_remotes()
except TypeError:
# Handle malformed ext_pillar
pass
if 'ext_pillar' not in self.opts:
return pillar, errors
if not isinstance(self.opts['ext_pillar'], list):

View file

@ -5334,7 +5334,8 @@ def serialize(name,
mode=None,
backup='',
makedirs=False,
show_diff=True,
show_diff=None,
show_changes=True,
create=True,
merge_if_exists=False,
**kwargs):
@ -5386,7 +5387,14 @@ def serialize(name,
.. versionadded:: 2014.1.3
show_diff
If set to False, the diff will not be shown.
DEPRECATED: Please use show_changes.
If set to ``False``, the diff will not be shown in the return data if
changes are made.
show_changes
Output a unified diff of the old file and the new file. If ``False``
return a boolean if any changes were made.
create
Default is True, if create is set to False then the file will only be
@ -5518,6 +5526,14 @@ def serialize(name,
# Make sure that any leading zeros stripped by YAML loader are added back
mode = salt.utils.normalize_mode(mode)
if show_diff is not None:
show_changes = show_diff
msg = (
'The \'show_diff\' argument to the file.serialized state has been '
'deprecated, please use \'show_changes\' instead.'
)
salt.utils.warn_until('Oxygen', msg)
if __opts__['test']:
ret['changes'] = __salt__['file.check_managed_changes'](
name=name,
@ -5540,10 +5556,12 @@ def serialize(name,
ret['result'] = None
ret['comment'] = 'Dataset will be serialized and stored into {0}'.format(
name)
if not show_changes:
ret['changes']['diff'] = '<show_changes=False>'
else:
ret['result'] = True
ret['comment'] = 'The file {0} is in the correct state'.format(name)
return ret
return __salt__['file.manage_file'](name=name,
@ -5558,7 +5576,7 @@ def serialize(name,
backup=backup,
makedirs=makedirs,
template=None,
show_changes=show_diff,
show_changes=show_changes,
contents=contents)

View file

@ -1915,7 +1915,7 @@ def latest(
for pkg in desired_pkgs:
if not avail.get(pkg):
# Package either a) is up-to-date, or b) does not exist
if not cur[pkg]:
if not cur.get(pkg):
# Package does not exist
msg = 'No information found for \'{0}\'.'.format(pkg)
log.error(msg)

View file

@ -1094,10 +1094,10 @@ def format_call(fun,
continue
extra[key] = copy.deepcopy(value)
# We'll be showing errors to the users until Salt Nitrogen comes out, after
# We'll be showing errors to the users until Salt Oxygen comes out, after
# which, errors will be raised instead.
warn_until(
'Nitrogen',
'Oxygen',
'It\'s time to start raising `SaltInvocationError` instead of '
'returning warnings',
# Let's not show the deprecation warning on the console, there's no
@ -1134,7 +1134,7 @@ def format_call(fun,
'{0}. If you were trying to pass additional data to be used '
'in a template context, please populate \'context\' with '
'\'key: value\' pairs. Your approach will work until Salt '
'Nitrogen is out.{1}'.format(
'Oxygen is out.{1}'.format(
msg,
'' if 'full' not in ret else ' Please update your state files.'
)

View file

@ -31,7 +31,7 @@ class Depends(object):
cause the function to be unloaded (or replaced)
'''
# kind -> Dependency -> list of things that depend on it
dependency_dict = defaultdict(lambda: defaultdict(set))
dependency_dict = defaultdict(lambda: defaultdict(dict))
def __init__(self, *dependencies, **kwargs):
'''
@ -72,11 +72,11 @@ class Depends(object):
frame = inspect.stack()[1][0]
# due to missing *.py files under esky we cannot use inspect.getmodule
# module name is something like salt.loaded.int.modules.test
kind = frame.f_globals['__name__'].rsplit('.', 2)[1]
_, kind, mod_name = frame.f_globals['__name__'].rsplit('.', 2)
fun_name = function.__name__
for dep in self.dependencies:
self.dependency_dict[kind][dep].add(
(frame, function, self.fallback_function)
)
self.dependency_dict[kind][dep][(mod_name, fun_name)] = \
(frame, self.fallback_function)
except Exception as exc:
log.error('Exception encountered when attempting to inspect frame in '
'dependency decorator: {0}'.format(exc))
@ -90,46 +90,44 @@ class Depends(object):
It will modify the "functions" dict and remove/replace modules that
are missing dependencies.
'''
for dependency, dependent_set in six.iteritems(cls.dependency_dict[kind]):
# check if dependency is loaded
for frame, func, fallback_function in dependent_set:
# check if you have the dependency
for dependency, dependent_dict in six.iteritems(cls.dependency_dict[kind]):
for (mod_name, func_name), (frame, fallback_function) in six.iteritems(dependent_dict):
# check if dependency is loaded
if dependency is True:
log.trace(
'Dependency for {0}.{1} exists, not unloading'.format(
frame.f_globals['__name__'].split('.')[-1],
func.__name__,
mod_name,
func_name
)
)
continue
# check if you have the dependency
if dependency in frame.f_globals \
or dependency in frame.f_locals:
log.trace(
'Dependency ({0}) already loaded inside {1}, '
'skipping'.format(
dependency,
frame.f_globals['__name__'].split('.')[-1]
mod_name
)
)
continue
log.trace(
'Unloading {0}.{1} because dependency ({2}) is not '
'imported'.format(
frame.f_globals['__name__'],
func,
mod_name,
func_name,
dependency
)
)
# if not, unload dependent_set
# if not, unload the function
if frame:
try:
func_name = frame.f_globals['__func_alias__'][func.__name__]
func_name = frame.f_globals['__func_alias__'][func_name]
except (AttributeError, KeyError):
func_name = func.__name__
pass
mod_key = '{0}.{1}'.format(frame.f_globals['__name__'].split('.')[-1],
func_name)
mod_key = '{0}.{1}'.format(mod_name, func_name)
# if we don't have this module loaded, skip it!
if mod_key not in functions:

View file

@ -702,8 +702,17 @@ class SignalHandlingMultiprocessingProcess(MultiprocessingProcess):
def default_signals(*signals):
old_signals = {}
for signum in signals:
old_signals[signum] = signal.getsignal(signum)
signal.signal(signum, signal.SIG_DFL)
try:
signal.signal(signum, signal.SIG_DFL)
old_signals[signum] = signal.getsignal(signum)
except ValueError as exc:
# This happens when a netapi module attempts to run a function
# using wheel_async, because the process trying to register signals
# will not be the main PID.
log.trace(
'Failed to register signal for signum %d: %s',
signum, exc
)
# Do whatever is needed with the reset signals
yield

View file

@ -147,7 +147,6 @@ def query(key, keyid, method='GET', params=None, headers=None,
if not data:
data = None
response = None
if method == 'PUT':
if local_file:
data = salt.utils.fopen(local_file, 'r')
@ -165,6 +164,7 @@ def query(key, keyid, method='GET', params=None, headers=None,
data=data,
verify=verify_ssl,
stream=True)
response = result.content
else:
result = requests.request(method,
requesturl,

View file

@ -517,8 +517,9 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin):
'''
Tests describing parameters via vpc id if vpc exist
'''
# With moto 0.4.25 is_default is set to True. 0.4.24 and older, is_default is False
if _get_moto_version() >= LooseVersion('0.4.25'):
# With moto 0.4.25 through 0.4.30, is_default is set to True.
# 0.4.24 and older and 0.4.31 and newer, is_default is False
if LooseVersion('0.4.25') <= _get_moto_version() < LooseVersion('0.4.31'):
is_default = True
else:
is_default = False

View file

@ -117,6 +117,7 @@ class ClearReqTestCases(BaseZMQReqCase, ReqChannelMixin):
raise tornado.gen.Return((payload, {'fun': 'send_clear'}))
@skipIf(True, 'Skipping flaky test until Jenkins is moved to C7.')
@skipIf(ON_SUSE, 'Skipping until https://github.com/saltstack/salt/issues/32902 gets fixed')
class AESReqTestCases(BaseZMQReqCase, ReqChannelMixin):
def setUp(self):
@ -142,6 +143,9 @@ class AESReqTestCases(BaseZMQReqCase, ReqChannelMixin):
'''
Test a variety of bad requests, make sure that we get some sort of error
'''
# TODO: This test should be re-enabled when Jenkins moves to C7.
# Once the version of salt-testing is increased to something newer than the September
# release of salt-testing, the @flaky decorator should be applied to this test.
msgs = ['', [], tuple()]
for msg in msgs:
with self.assertRaises(salt.exceptions.AuthenticationError):