saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge branch '2018.3' into 2018.3-remove-trailing-commas-on-linux-user-gecos-fields

Browse source
This commit is contained in:
Nicole Thomas 2018-05-22 11:35:20 -04:00 • committed by GitHub
commit 60ec3230db
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
35 changed files with 421 additions and 132 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
doc/man/salt.7
View file

@ -341754,7 +341754,6 @@ netacl_example:
Or:
.INDENT 7.0
.INDENT 3.5
.sp
.nf
.ft C
netacl_example:

28
doc/ref/configuration/minion.rst
View file

@ -3219,3 +3219,31 @@ URL of the repository:
Replace ``<commit_id>`` with the SHA1 hash of a commit ID. Specifying a commit
ID is useful in that it allows one to revert back to a previous version in the
event that an error is introduced in the latest revision of the repo.
``ssh_merge_pillar``
--------------------
.. versionadded:: 2018.3.2
Default: ``True``
Merges the compiled pillar data with the pillar data already available globally.
This is useful when using ``salt-ssh`` or ``salt-call --local`` and overriding the pillar
data in a state file:
.. code-block:: yaml
apply_showpillar:
module.run:
- name: state.apply
- mods:
- showpillar
- kwargs:
pillar:
test: "foo bar"
If set to ``True`` the ``showpillar`` state will have access to the
global pillar data.
If set to ``False`` only the overriding pillar data will be available
to the ``showpillar`` state.

51
doc/topics/beacons/index.rst
View file

@ -40,8 +40,9 @@ Beacons are typically enabled by placing a ``beacons:`` top level block in
beacons:
inotify:
/etc/important_file: {}
/opt: {}
- files:
/etc/important_file: {}
/opt: {}
The beacon system, like many others in Salt, can also be configured via the
minion pillar, grains, or local config file.
@ -50,6 +51,8 @@ minion pillar, grains, or local config file.
The `inotify` beacon only works on OSes that have `inotify` kernel support.
Currently this excludes FreeBSD, macOS, and Windows.
All beacon configuration is done using list based configuration.
Beacon Monitoring Interval
--------------------------
@ -61,21 +64,23 @@ and 10-second intervals:
beacons:
inotify:
/etc/important_file: {}
/opt: {}
interval: 5
disable_during_state_run: True
- files:
/etc/important_file: {}
/opt: {}
- interval: 5
- disable_during_state_run: True
load:
1m:
- 0.0
- 2.0
5m:
- 0.0
- 1.5
15m:
- 0.1
- 1.0
interval: 10
- averages:
1m:
- 0.0
- 2.0
5m:
- 0.0
- 1.5
15m:
- 0.1
- 1.0
- interval: 10
.. _avoid-beacon-event-loops:
@ -96,8 +101,9 @@ which point the normal beacon interval will resume.
beacons:
inotify:
/etc/important_file: {}
disable_during_state_run: True
- files:
/etc/important_file: {}
- disable_during_state_run: True
.. _beacon-example:
@ -137,10 +143,11 @@ On the Salt minion, add the following configuration to
beacons:
inotify:
/etc/important_file:
mask:
- modify
disable_during_state_run: True
- files:
/etc/important_file:
mask:
- modify
- disable_during_state_run: True
Save the configuration file and restart the minion service. The beacon is now
set up to notify salt upon modifications made to the file.

14
doc/topics/installation/debian.rst
View file

@ -6,7 +6,7 @@ Debian GNU/Linux / Raspbian
Debian GNU/Linux distribution and some derivatives such as Raspbian already
have included Salt packages to their repositories. However, current stable
release codenamed "Jessie" contains old outdated Salt release. It is
Debian release contains old outdated Salt releases. It is
recommended to use SaltStack repository for Debian as described
:ref:`below <installation-debian-repo>`.
@ -33,11 +33,13 @@ Instructions are at https://repo.saltstack.com/#debian.
Installation from the Debian / Raspbian Official Repository
===========================================================
Stretch (Testing) and Sid (Unstable) distributions are already contain mostly
up-to-date Salt packages built by Debian Salt Team. You can install Salt
components directly from Debian.
The Debian distributions contain mostly old Salt packages
built by the Debian Salt Team. You can install Salt
components directly from Debian but it is recommended to
use the instructions above for the packages from the official
Salt repository.
On Jessie (Stable) there is an option to install Salt minion from Stretch with
On Jessie there is an option to install Salt minion from Stretch with
`python-tornado` dependency from `jessie-backports` repositories.
To install fresh release of Salt minion on Jessie:
@ -79,7 +81,7 @@ To install fresh release of Salt minion on Jessie:
apt-get update
apt-get install python-zmq python-tornado/stretch salt-common/stretch
#. Install Salt minion package from Stretch:
#. Install Salt minion package from Latest Debian Release:
.. code-block:: bash

4
requirements/base.txt
View file

@ -1,5 +1,7 @@
Jinja2
msgpack-python>0.3,!=0.5.5
# This should be changed to msgpack-python for Packages
# msgpack-python>0.3,!=0.5.5
msgpack>=0.5,!=0.5.5
PyYAML
MarkupSafe
requests>=1.0.0

4
salt/config/__init__.py
View file

@ -989,6 +989,7 @@ VALID_OPTS = {
'ssh_identities_only': bool,
'ssh_log_file': six.string_types,
'ssh_config_file': six.string_types,
'ssh_merge_pillar': bool,
# Enable ioflo verbose logging. Warning! Very verbose!
'ioflo_verbose': int,
@ -1485,6 +1486,7 @@ DEFAULT_MINION_OPTS = {
},
'discovery': False,
'schedule': {},
'ssh_merge_pillar': True
}
DEFAULT_MASTER_OPTS = {
@ -2088,7 +2090,7 @@ def _validate_ssh_minion_opts(opts):
for opt_name in list(ssh_minion_opts):
if re.match('^[a-z0-9]+fs_', opt_name, flags=re.IGNORECASE) \
or 'pillar' in opt_name \
or ('pillar' in opt_name and not 'ssh_merge_pillar' == opt_name) \
or opt_name in ('fileserver_backend',):
log.warning(
'\'%s\' is not a valid ssh_minion_opts parameter, ignoring',

3
salt/grains/core.py
View file

@ -1213,6 +1213,7 @@ _OS_NAME_MAP = {
'synology': 'Synology',
'nilrt': 'NILinuxRT',
'nilrt-xfce': 'NILinuxRT-XFCE',
'poky': 'Poky',
'manjaro': 'Manjaro',
'manjarolin': 'Manjaro',
'antergos': 'Antergos',
@ -1766,7 +1767,7 @@ def os_data():
osarch = __salt__['cmd.run']('dpkg --print-architecture').strip()
elif grains.get('os_family') == 'RedHat':
osarch = __salt__['cmd.run']('rpm --eval %{_host_cpu}').strip()
elif grains.get('os_family') == 'NILinuxRT':
elif grains.get('os_family') in ('NILinuxRT', 'Poky'):
archinfo = {}
for line in __salt__['cmd.run']('opkg print-architecture').splitlines():
if line.startswith('arch'):

2
salt/grains/napalm.py
View file

@ -91,7 +91,7 @@ def _retrieve_device_cache(proxy=None):
DEVICE_CACHE = proxy['napalm.get_device']()
elif not proxy and salt.utils.napalm.is_minion(__opts__):
# if proxy var not passed and is running in a straight minion
DEVICE_CACHE = salt.utils.napalm.get_device_opts(__opts__)
DEVICE_CACHE = salt.utils.napalm.get_device(__opts__)
return DEVICE_CACHE

6
salt/loader.py
View file

@ -1629,7 +1629,11 @@ class LazyLoader(salt.utils.lazy.LazyDict):
return True
# if the modulename isn't in the whitelist, don't bother
if self.whitelist and mod_name not in self.whitelist:
raise KeyError
log.error(
'Failed to load function %s because its module (%s) is '
'not in the whitelist: %s', key, mod_name, self.whitelist
)
raise KeyError(key)
def _inner_load(mod_name):
for name in self._iter_files(mod_name):

14
salt/log/handlers/__init__.py
View file

@ -103,6 +103,20 @@ class SysLogHandler(ExcInfoOnLogLevelFormatMixIn, logging.handlers.SysLogHandler
'''
Syslog handler which properly handles exc_info on a per handler basis
'''
def handleError(self, record):
'''
Override the default error handling mechanism for py3
Deal with syslog os errors when the log file does not exist
'''
handled = False
if sys.stderr and sys.version_info >= (3, 5, 4):
t, v, tb = sys.exc_info()
if t.__name__ in 'FileNotFoundError':
sys.stderr.write('[WARNING ] The log_file does not exist. Logging not setup correctly or syslog service not started.\n')
handled = True
if not handled:
super(SysLogHandler, self).handleError(record)
class RotatingFileHandler(ExcInfoOnLogLevelFormatMixIn, logging.handlers.RotatingFileHandler, NewStyleClassMixIn):

12
salt/log/handlers/sentry_mod.py
View file

@ -5,7 +5,12 @@
.. versionadded:: 0.17.0
This module provides a `Sentry`_ logging handler.
This module provides a `Sentry`_ logging handler. Sentry is an open source
error tracking platform that provides deep context about exceptions that
happen in production. Details about stack traces along with the context
variables available at the time of the exeption are easily browsable and
filterable from the online interface. For more details please see
`Sentry`_.
.. admonition:: Note
@ -41,6 +46,11 @@
- cpuarch
- ec2.tags.environment
.. admonition:: Note
The ``public_key`` and ``secret_key`` variables are not supported with
Sentry > 3.0. The `DSN`_ key should be used instead.
All the client configuration keys are supported, please see the
`Raven client documentation`_.

6
salt/modules/archive.py
View file

@ -410,9 +410,9 @@ def list_(name,
item.sort()
if verbose:
ret = {'dirs': sorted(dirs),
'files': sorted(files),
'links': sorted(links)}
ret = {'dirs': sorted(salt.utils.data.decode_list(dirs)),
'files': sorted(salt.utils.data.decode_list(files)),
'links': sorted(salt.utils.data.decode_list(links))}
ret['top_level_dirs'] = [x for x in ret['dirs']
if x.count('/') == 1]
ret['top_level_files'] = [x for x in ret['files']

15
salt/modules/cmdmod.py
View file

@ -43,7 +43,7 @@ from salt.ext import six
from salt.exceptions import CommandExecutionError, TimedProcTimeoutError, \
SaltInvocationError
from salt.log import LOG_LEVELS
from salt.ext.six.moves import range, zip
from salt.ext.six.moves import range, zip, map
# Only available on POSIX systems, nonfatal on windows
try:
@ -405,6 +405,19 @@ def _run(cmd,
return win_runas(cmd, runas, password, cwd)
if runas and salt.utils.platform.is_darwin():
# we need to insert the user simulation into the command itself and not
# just run it from the environment on macOS as that
# method doesn't work properly when run as root for certain commands.
if isinstance(cmd, (list, tuple)):
cmd = ' '.join(map(_cmd_quote, cmd))
cmd = 'su -l {0} -c "{1}"'.format(runas, cmd)
# set runas to None, because if you try to run `su -l` as well as
# simulate the environment macOS will prompt for the password of the
# user and will cause salt to hang.
runas = None
if runas:
# Save the original command before munging it
try:

4
salt/modules/opkg.py
View file

@ -60,9 +60,9 @@ def __virtual__():
'''
Confirm this module is on a nilrt based system
'''
if __grains__.get('os_family', False) == 'NILinuxRT':
if os.path.isdir(OPKG_CONFDIR):
return __virtualname__
return (False, "Module opkg only works on nilrt based systems")
return False, "Module opkg only works on OpenEmbedded based systems"
def latest_version(*names, **kwargs):

133
salt/modules/rpmbuild.py
View file

@ -65,21 +65,22 @@ def __virtual__():
# The module will be exposed as `rpmbuild` on non-RPM based systems
return 'rpmbuild'
else:
return False, 'The rpmbuild module could not be loaded: requires python-gnupg, gpg, rpm, rpmbuild, mock and createrepo utilities to be installed'
return False, 'The rpmbuild module could not be loaded: requires python-gnupg, ' \
'gpg, rpm, rpmbuild, mock and createrepo utilities to be installed'
def _create_rpmmacros():
def _create_rpmmacros(runas='root'):
'''
Create the .rpmmacros file in user's home directory
'''
home = os.path.expanduser('~')
rpmbuilddir = os.path.join(home, 'rpmbuild')
if not os.path.isdir(rpmbuilddir):
os.makedirs(rpmbuilddir)
__salt__['file.makedirs_perms'](name=rpmbuilddir, user=runas, group='mock')
mockdir = os.path.join(home, 'mock')
if not os.path.isdir(mockdir):
os.makedirs(mockdir)
__salt__['file.makedirs_perms'](name=mockdir, user=runas, group='mock')
rpmmacros = os.path.join(home, '.rpmmacros')
with salt.utils.files.fopen(rpmmacros, 'w') as afile:
@ -92,7 +93,7 @@ def _create_rpmmacros():
afile.write('%_gpg_name packaging@saltstack.com\n')
def _mk_tree():
def _mk_tree(runas='root'):
'''
Create the rpm build tree
'''
@ -100,7 +101,7 @@ def _mk_tree():
paths = ['BUILD', 'RPMS', 'SOURCES', 'SPECS', 'SRPMS']
for path in paths:
full = os.path.join(basedir, path)
os.makedirs(full)
__salt__['file.makedirs_perms'](name=full, user=runas, group='mock')
return basedir
@ -116,7 +117,7 @@ def _get_spec(tree_base, spec, template, saltenv='base'):
saltenv=saltenv)
def _get_src(tree_base, source, saltenv='base'):
def _get_src(tree_base, source, saltenv='base', runas='root'):
'''
Get the named sources and place them into the tree_base
'''
@ -127,6 +128,7 @@ def _get_src(tree_base, source, saltenv='base'):
lsrc = __salt__['cp.get_url'](source, dest, saltenv=saltenv)
else:
shutil.copy(source, dest)
__salt__['file.chown'](path=dest, user=runas, group='mock')
def _get_distset(tgt):
@ -171,7 +173,7 @@ def _get_deps(deps, tree_base, saltenv='base'):
return deps_list
def make_src_pkg(dest_dir, spec, sources, env=None, template=None, saltenv='base'):
def make_src_pkg(dest_dir, spec, sources, env=None, template=None, saltenv='base', runas='root'):
'''
Create a source rpm from the given spec file and sources
@ -179,33 +181,74 @@ def make_src_pkg(dest_dir, spec, sources, env=None, template=None, saltenv='base
.. code-block:: bash
salt '*' pkgbuild.make_src_pkg /var/www/html/ https://raw.githubusercontent.com/saltstack/libnacl/master/pkg/rpm/python-libnacl.spec https://pypi.python.org/packages/source/l/libnacl/libnacl-1.3.5.tar.gz
salt '*' pkgbuild.make_src_pkg /var/www/html/
https://raw.githubusercontent.com/saltstack/libnacl/master/pkg/rpm/python-libnacl.spec
https://pypi.python.org/packages/source/l/libnacl/libnacl-1.3.5.tar.gz
This example command should build the libnacl SOURCE package and place it in
/var/www/html/ on the minion
.. versionchanged:: 2017.7.0
dest_dir
The directory on the minion to place the built package(s)
spec
The location of the spec file (used for rpms)
sources
The list of package sources
env
A dictionary of environment variables to be set prior to execution.
template
Run the spec file through a templating engine
Optional arguement, allows for no templating engine used to be
if none is desired.
saltenv
The saltenv to use for files downloaded from the salt filesever
runas
The user to run the build process as
.. versionadded:: 2018.3.2
.. note::
using SHA256 as digest and minimum level dist el6
'''
_create_rpmmacros()
tree_base = _mk_tree()
_create_rpmmacros(runas)
tree_base = _mk_tree(runas)
spec_path = _get_spec(tree_base, spec, template, saltenv)
__salt__['file.chown'](path=spec_path, user=runas, group='mock')
__salt__['file.chown'](path=tree_base, user=runas, group='mock')
if isinstance(sources, six.string_types):
sources = sources.split(',')
for src in sources:
_get_src(tree_base, src, saltenv)
_get_src(tree_base, src, saltenv, runas)
# make source rpms for dist el6 with SHA256, usable with mock on other dists
cmd = 'rpmbuild --verbose --define "_topdir {0}" -bs --define "dist .el6" {1}'.format(tree_base, spec_path)
__salt__['cmd.run'](cmd)
retrc = __salt__['cmd.retcode'](cmd, runas=runas)
if retrc != 0:
raise SaltInvocationError(
'Make source package for destination directory {0}, spec {1}, sources {2}, failed '
'with return error {3}, check logs for further details'.format(
dest_dir,
spec,
sources,
retrc)
)
srpms = os.path.join(tree_base, 'SRPMS')
ret = []
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
__salt__['file.makedirs_perms'](name=dest_dir, user=runas, group='mock')
for fn_ in os.listdir(srpms):
full = os.path.join(srpms, fn_)
tgt = os.path.join(dest_dir, fn_)
@ -232,14 +275,16 @@ def build(runas,
.. code-block:: bash
salt '*' pkgbuild.build mock epel-7-x86_64 /var/www/html https://raw.githubusercontent.com/saltstack/libnacl/master/pkg/rpm/python-libnacl.spec https://pypi.python.org/packages/source/l/libnacl/libnacl-1.3.5.tar.gz
salt '*' pkgbuild.build mock epel-7-x86_64 /var/www/html
https://raw.githubusercontent.com/saltstack/libnacl/master/pkg/rpm/python-libnacl.spec
https://pypi.python.org/packages/source/l/libnacl/libnacl-1.3.5.tar.gz
This example command should build the libnacl package for rhel 7 using user
mock and place it in /var/www/html/ on the minion
'''
ret = {}
try:
os.makedirs(dest_dir)
__salt__['file.chown'](path=dest_dir, user=runas, group='mock')
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
@ -247,7 +292,7 @@ def build(runas,
srpm_build_dir = tempfile.mkdtemp()
try:
srpms = make_src_pkg(srpm_build_dir, spec, sources,
env, template, saltenv)
env, template, saltenv, runas)
except Exception as exc:
shutil.rmtree(srpm_build_dir)
log.error('Failed to make src package')
@ -259,17 +304,18 @@ def build(runas,
deps_dir = tempfile.mkdtemp()
deps_list = _get_deps(deps, deps_dir, saltenv)
retrc = 0
for srpm in srpms:
dbase = os.path.dirname(srpm)
results_dir = tempfile.mkdtemp()
try:
__salt__['cmd.run']('chown {0} -R {1}'.format(runas, dbase))
__salt__['cmd.run']('chown {0} -R {1}'.format(runas, results_dir))
__salt__['file.chown'](path=dbase, user=runas, group='mock')
__salt__['file.chown'](path=results_dir, user=runas, group='mock')
cmd = 'mock --root={0} --resultdir={1} --init'.format(tgt, results_dir)
__salt__['cmd.run'](cmd, runas=runas)
retrc |= __salt__['cmd.retcode'](cmd, runas=runas)
if deps_list and not deps_list.isspace():
cmd = 'mock --root={0} --resultdir={1} --install {2} {3}'.format(tgt, results_dir, deps_list, noclean)
__salt__['cmd.run'](cmd, runas=runas)
retrc |= __salt__['cmd.retcode'](cmd, runas=runas)
noclean += ' --no-clean'
cmd = 'mock --root={0} --resultdir={1} {2} {3} {4}'.format(
@ -278,17 +324,20 @@ def build(runas,
distset,
noclean,
srpm)
__salt__['cmd.run'](cmd, runas=runas)
cmd = ['rpm', '-qp', '--queryformat',
'{0}/%{{name}}/%{{version}}-%{{release}}'.format(log_dir),
srpm]
log_dest = __salt__['cmd.run_stdout'](cmd, python_shell=False)
retrc |= __salt__['cmd.retcode'](cmd, runas=runas)
cmdlist = [
'rpm',
'-qp',
'--queryformat',
'{0}/%{{name}}/%{{version}}-%{{release}}'.format(log_dir),
srpm]
log_dest = __salt__['cmd.run_stdout'](cmdlist, python_shell=False)
for filename in os.listdir(results_dir):
full = os.path.join(results_dir, filename)
if filename.endswith('src.rpm'):
sdest = os.path.join(srpm_dir, filename)
try:
os.makedirs(srpm_dir)
__salt__['file.makedirs_perms'](name=srpm_dir, user=runas, group='mock')
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
@ -301,7 +350,7 @@ def build(runas,
else:
log_file = os.path.join(log_dest, filename)
try:
os.makedirs(log_dest)
__salt__['file.makedirs_perms'](name=log_dest, user=runas, group='mock')
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
@ -311,6 +360,15 @@ def build(runas,
log.error('Error building from %s: %s', srpm, exc)
finally:
shutil.rmtree(results_dir)
if retrc != 0:
raise SaltInvocationError(
'Building packages for destination directory {0}, spec {1}, sources {2}, failed '
'with return error {3}, check logs for further details'.format(
dest_dir,
spec,
sources,
retrc)
)
shutil.rmtree(deps_dir)
shutil.rmtree(srpm_build_dir)
return ret
@ -433,7 +491,7 @@ def make_repo(repodir,
phrase = ''
if keyid is not None:
## import_keys
# import_keys
pkg_pub_key_file = '{0}/{1}'.format(gnupghome, __salt__['pillar.get']('gpg_pkg_pub_keyname', None))
pkg_priv_key_file = '{0}/{1}'.format(gnupghome, __salt__['pillar.get']('gpg_pkg_priv_keyname', None))
@ -477,14 +535,21 @@ def make_repo(repodir,
# need to update rpm with public key
cmd = 'rpm --import {0}'.format(pkg_pub_key_file)
__salt__['cmd.run'](cmd, runas=runas, use_vt=True)
retrc = __salt__['cmd.retcode'](cmd, runas=runas, use_vt=True)
if retrc != 0:
raise SaltInvocationError(
'Failed to import public key from file {0} with return '
'error {1}, check logs for further details'.format(
pkg_pub_key_file,
retrc)
)
## sign_it_here
# sign_it_here
# interval of 0.125 is really too fast on some systems
interval = 0.5
for file in os.listdir(repodir):
if file.endswith('.rpm'):
abs_file = os.path.join(repodir, file)
for fileused in os.listdir(repodir):
if fileused.endswith('.rpm'):
abs_file = os.path.join(repodir, fileused)
number_retries = timeout / interval
times_looped = 0
error_msg = 'Failed to sign file {0}'.format(abs_file)

13
salt/output/table_out.py
View file

@ -43,6 +43,7 @@ from functools import reduce # pylint: disable=redefined-builtin
# Import Salt libs
import salt.output
import salt.utils.color
import salt.utils.data
import salt.utils.locales
# Import 3rd-party libs
@ -62,9 +63,9 @@ class TableDisplay(object):
'''
_JUSTIFY_MAP = {
'center': str.center,
'right': str.rjust,
'left': str.ljust
'center': six.text_type.center,
'right': six.text_type.rjust,
'left': six.text_type.ljust
}
def __init__(self,
@ -147,7 +148,7 @@ class TableDisplay(object):
for item in row
]
rows = []
for item in map(None, *new_rows):
for item in map(lambda *args: args, *new_rows):
if isinstance(item, (tuple, list)):
rows.append([substr or '' for substr in item])
else:
@ -159,7 +160,7 @@ class TableDisplay(object):
for row in rows
]
columns = map(None, *reduce(operator.add, logical_rows))
columns = map(lambda *args: args, *reduce(operator.add, logical_rows))
max_widths = [
max([len(six.text_type(item)) for item in column])
@ -363,7 +364,7 @@ def output(ret, **kwargs):
)
)
return '\n'.join(table.display(ret,
return '\n'.join(table.display(salt.utils.data.decode(ret),
base_indent,
out,
rows_key=rows_key,

7
salt/pillar/__init__.py
View file

@ -1014,6 +1014,13 @@ class Pillar(object):
mopts['file_roots'] = self.actual_file_roots
mopts['saltversion'] = __version__
pillar['master'] = mopts
if 'pillar' in self.opts and self.opts.get('ssh_merge_pillar', False):
pillar = merge(
self.opts['pillar'],
pillar,
self.merge_strategy,
self.opts.get('renderer', 'yaml'),
self.opts.get('pillar_merge_lists', False))
if errors:
for error in errors:
log.critical('Pillar render error: %s', error)

11
salt/proxy/napalm.py
View file

@ -236,16 +236,7 @@ def get_grains():
'''
Retrieve facts from the network device.
'''
refresh_needed = False
refresh_needed = refresh_needed or (not DETAILS.get('grains_cache', {}))
refresh_needed = refresh_needed or (not DETAILS.get('grains_cache', {}).get('result', False))
refresh_needed = refresh_needed or (not DETAILS.get('grains_cache', {}).get('out', {}))
if refresh_needed:
facts = call('get_facts', **{})
DETAILS['grains_cache'] = facts
return DETAILS.get('grains_cache', {})
return call('get_facts', **{})
def grains_refresh():

4
salt/states/archive.py
View file

@ -1090,7 +1090,7 @@ def extracted(name,
and not stat.S_ISDIR(x)),
(contents['links'], stat.S_ISLNK)):
for path in path_list:
full_path = os.path.join(name, path)
full_path = salt.utils.path.join(name, path)
try:
path_mode = os.lstat(full_path.rstrip(os.sep)).st_mode
if not func(path_mode):
@ -1259,7 +1259,7 @@ def extracted(name,
if options is None:
try:
with closing(tarfile.open(cached, 'r')) as tar:
tar.extractall(name)
tar.extractall(salt.utils.stringutils.to_str(name))
files = tar.getnames()
if trim_output:
files = files[:trim_output]

3
salt/states/docker_image.py
View file

@ -349,9 +349,10 @@ def present(name,
# Only add to the changes dict if layers were pulled
ret['changes'] = image_update
error = False
try:
__salt__['docker.inspect_image'](full_image)
error = False
except CommandExecutionError as exc:
msg = exc.__str__()
if '404' not in msg:

9
salt/states/mysql_query.py
View file

@ -57,10 +57,13 @@ def run_file(name,
grain=None,
key=None,
overwrite=True,
saltenv=None,
**connection_args):
'''
Execute an arbitrary query on the specified database
.. versionadded:: 2017.7.0
name
Used only as an ID
@ -85,13 +88,17 @@ def run_file(name,
overwrite:
The file or grain will be overwritten if it already exists (default)
.. versionadded:: 2017.7.0
saltenv:
The saltenv to pull the query_file from
'''
ret = {'name': name,
'changes': {},
'result': True,
'comment': 'Database {0} is already present'.format(database)}
if any([query_file.startswith(proto) for proto in ['http://', 'https://', 'salt://', 's3://', 'swift://']]):
query_file = __salt__['cp.cache_file'](query_file, saltenv=saltenv or __env__)
if not os.path.exists(query_file):
ret['comment'] = 'File {0} does not exist'.format(query_file)
ret['result'] = False

17
salt/states/pkgrepo.py
View file

@ -359,7 +359,6 @@ def managed(name, ppa=None, **kwargs):
enabled = True
repo = name
os_family = __grains__['os_family'].lower()
if __grains__['os'] in ('Ubuntu', 'Mint'):
if ppa is not None:
# overload the name/repo value for PPAs cleanly
@ -373,7 +372,7 @@ def managed(name, ppa=None, **kwargs):
if enabled is not None \
else salt.utils.data.is_true(disabled)
elif os_family in ('redhat', 'suse'):
elif __grains__['os_family'] in ('RedHat', 'Suse'):
if 'humanname' in kwargs:
kwargs['name'] = kwargs.pop('humanname')
if 'name' not in kwargs:
@ -384,7 +383,7 @@ def managed(name, ppa=None, **kwargs):
if disabled is not None \
else salt.utils.data.is_true(enabled)
elif os_family == 'nilinuxrt':
elif __grains__['os_family'] in ('NILinuxRT', 'Poky'):
# opkg is the pkg virtual
kwargs['enabled'] = not salt.utils.data.is_true(disabled) \
if disabled is not None \
@ -413,7 +412,7 @@ def managed(name, ppa=None, **kwargs):
else:
sanitizedkwargs = kwargs
if os_family == 'debian':
if __grains__['os_family'] == 'Debian':
repo = salt.utils.pkg.deb.strip_uri(repo)
if pre:
@ -427,7 +426,7 @@ def managed(name, ppa=None, **kwargs):
# not explicitly set, so we don't need to update the repo
# if it's desired to be enabled and the 'enabled' key is
# missing from the repo definition
if os_family == 'redhat':
if __grains__['os_family'] == 'RedHat':
if not salt.utils.data.is_true(sanitizedkwargs[kwarg]):
break
else:
@ -437,7 +436,7 @@ def managed(name, ppa=None, **kwargs):
elif kwarg == 'comps':
if sorted(sanitizedkwargs[kwarg]) != sorted(pre[kwarg]):
break
elif kwarg == 'line' and os_family == 'debian':
elif kwarg == 'line' and __grains__['os_family'] == 'Debian':
# split the line and sort everything after the URL
sanitizedsplit = sanitizedkwargs[kwarg].split()
sanitizedsplit[3:] = sorted(sanitizedsplit[3:])
@ -452,14 +451,14 @@ def managed(name, ppa=None, **kwargs):
salt.utils.pkg.deb.combine_comments(kwargs['comments'])
if pre_comments != post_comments:
break
elif kwarg == 'comments' and os_family == 'redhat':
elif kwarg == 'comments' and __grains__['os_family'] == 'RedHat':
precomments = salt.utils.pkg.rpm.combine_comments(pre[kwarg])
kwargcomments = salt.utils.pkg.rpm.combine_comments(
sanitizedkwargs[kwarg])
if precomments != kwargcomments:
break
else:
if os_family in ('redhat', 'suse') \
if __grains__['os_family'] in ('RedHat', 'Suse') \
and any(isinstance(x, bool) for x in
(sanitizedkwargs[kwarg], pre[kwarg])):
# This check disambiguates 1/0 from True/False
@ -490,7 +489,7 @@ def managed(name, ppa=None, **kwargs):
pass
try:
if os_family == 'debian':
if __grains__['os_family'] == 'Debian':
__salt__['pkg.mod_repo'](repo, saltenv=__env__, **kwargs)
else:
__salt__['pkg.mod_repo'](repo, **kwargs)

12
salt/utils/schedule.py
View file

@ -755,11 +755,13 @@ class Schedule(object):
else:
# Send back to master so the job is included in the job list
mret = ret.copy()
mret['jid'] = 'req'
if data.get('return_job') == 'nocache':
# overwrite 'req' to signal to master that
# this job shouldn't be stored
mret['jid'] = 'nocache'
# No returners defined, so we're only sending back to the master
if not data_returner and not self.schedule_returner:
mret['jid'] = 'req'
if data.get('return_job') == 'nocache':
# overwrite 'req' to signal to master that
# this job shouldn't be stored
mret['jid'] = 'nocache'
load = {'cmd': '_return', 'id': self.opts['id']}
for key, value in six.iteritems(mret):
load[key] = value

2
templates/module/salt/modules/{{module_name}}.py
View file

@ -14,7 +14,7 @@
'''
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals, print_function
import logging
# Import salt libs

2
templates/state/salt/states/{{module_name}}.py
View file

@ -15,7 +15,7 @@
'''
# Import Python libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals, print_function
import logging
# Import salt libs

2
templates/test_module/tests/unit/modules/test_{{module_name}}.py
View file

@ -4,7 +4,7 @@
'''
# Import Python Libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals, print_function
# Import Salt Testing Libs
from tests.support.mixins import LoaderModuleMockMixin

2
templates/test_state/tests/unit/states/test_{{module_name}}.py
View file

@ -4,7 +4,7 @@
'''
# Import Python Libs
from __future__ import absolute_import
from __future__ import absolute_import, unicode_literals, print_function
# Import Salt Testing Libs
from tests.support.mixins import LoaderModuleMockMixin

37
tests/integration/modules/test_archive.py
View file

@ -47,7 +47,7 @@ class ArchiveTest(ModuleCase):
self.arch = os.path.join(self.base_path, 'archive.{0}'.format(arch_fmt))
self.dst = os.path.join(self.base_path, '{0}_dst_dir'.format(arch_fmt))
def _set_up(self, arch_fmt):
def _set_up(self, arch_fmt, unicode_filename=False):
'''
Create source file tree and destination directory
@ -62,7 +62,11 @@ class ArchiveTest(ModuleCase):
# Create source
os.makedirs(self.src)
with salt.utils.files.fopen(os.path.join(self.src, 'file'), 'w') as theorem:
if unicode_filename:
filename = 'file®'
else:
filename = 'file'
with salt.utils.files.fopen(os.path.join(self.src, filename), 'w') as theorem:
theorem.write(textwrap.dedent(salt.utils.stringutils.to_str(r'''\
Compression theorem of computational complexity theory:
@ -150,6 +154,35 @@ class ArchiveTest(ModuleCase):
self._tear_down()
@skipIf(not salt.utils.path.which('tar'), 'Cannot find tar executable')
def test_tar_pack_unicode(self):
'''
Validate using the tar function to create archives
'''
self._set_up(arch_fmt='tar', unicode_filename=True)
# Test create archive
ret = self.run_function('archive.tar', ['-cvf', self.arch], sources=self.src)
self.assertTrue(isinstance(ret, list), six.text_type(ret))
self._assert_artifacts_in_ret(ret)
self._tear_down()
@skipIf(not salt.utils.path.which('tar'), 'Cannot find tar executable')
def test_tar_unpack_unicode(self):
'''
Validate using the tar function to extract archives
'''
self._set_up(arch_fmt='tar', unicode_filename=True)
self.run_function('archive.tar', ['-cvf', self.arch], sources=self.src)
# Test extract archive
ret = self.run_function('archive.tar', ['-xvf', self.arch], dest=self.dst)
self.assertTrue(isinstance(ret, list), six.text_type(ret))
self._assert_artifacts_in_ret(ret)
self._tear_down()
@skipIf(not salt.utils.path.which('gzip'), 'Cannot find gzip executable')
def test_gzip(self):
'''

4
tests/integration/modules/test_pip.py
View file

@ -11,7 +11,6 @@ from __future__ import absolute_import, print_function, unicode_literals
import os
import re
import shutil
import sys
import tempfile
# Import Salt Testing libs
@ -75,8 +74,7 @@ class PipModuleTest(ModuleCase):
# Let's remove the pip binary
pip_bin = os.path.join(self.venv_dir, 'bin', 'pip')
py_dir = 'python{0}.{1}'.format(*sys.version_info[:2])
site_dir = os.path.join(self.venv_dir, 'lib', py_dir, 'site-packages')
site_dir = self.run_function('virtualenv.get_distribution_path', [self.venv_dir, 'pip'])
if salt.utils.platform.is_windows():
pip_bin = os.path.join(self.venv_dir, 'Scripts', 'pip.exe')
site_dir = os.path.join(self.venv_dir, 'lib', 'site-packages')

42
tests/integration/shell/test_call.py
View file

@ -329,6 +329,48 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin
if os.path.isdir(config_dir):
shutil.rmtree(config_dir)
def test_syslog_file_not_found(self):
'''
test when log_file is set to a syslog file that does not exist
'''
old_cwd = os.getcwd()
config_dir = os.path.join(TMP, 'log_file_incorrect')
if not os.path.isdir(config_dir):
os.makedirs(config_dir)
os.chdir(config_dir)
with salt.utils.files.fopen(self.get_config_file_path('minion'), 'r') as fh_:
minion_config = salt.utils.yaml.load(fh_.read())
minion_config['log_file'] = 'file:///dev/doesnotexist'
with salt.utils.files.fopen(os.path.join(config_dir, 'minion'), 'w') as fh_:
fh_.write(
salt.utils.yaml.dump(minion_config, default_flow_style=False)
)
ret = self.run_script(
'salt-call',
'--config-dir {0} cmd.run "echo foo"'.format(
config_dir
),
timeout=60,
catch_stderr=True,
with_retcode=True
)
try:
if sys.version_info >= (3, 5, 4):
self.assertIn('local:', ret[0])
self.assertIn('[WARNING ] The log_file does not exist. Logging not setup correctly or syslog service not started.', ret[1])
self.assertEqual(ret[2], 0)
else:
self.assertIn(
'Failed to setup the Syslog logging handler', '\n'.join(ret[1])
)
self.assertEqual(ret[2], 2)
finally:
self.chdir(old_cwd)
if os.path.isdir(config_dir):
shutil.rmtree(config_dir)
def test_issue_15074_output_file_append(self):
output_file_append = os.path.join(TMP, 'issue-15074')
try:

26
tests/integration/states/test_pip_state.py
View file

@ -41,6 +41,7 @@ import salt.utils.platform
import salt.utils.versions
import salt.utils.win_dacl
import salt.utils.win_functions
import salt.utils.win_runas
from salt.modules.virtualenv_mod import KNOWN_BINARY_NAMES
from salt.exceptions import CommandExecutionError
@ -48,6 +49,26 @@ from salt.exceptions import CommandExecutionError
from salt.ext import six
def can_runas():
'''
Detect if we are running in a limited shell (winrm) and are un-able to use
the runas utility method.
'''
if salt.utils.platform.is_windows():
try:
salt.utils.win_runas.runas(
'cmd.exe /c echo 1', 'noexistuser', 'n0existp4ss',
)
except WindowsError as exc: # pylint: disable=undefined-variable
if exc.winerror == 5:
# Access Denied
return False
return True
CAN_RUNAS = can_runas()
class VirtualEnv(object):
def __init__(self, test, venv_dir):
self.venv_dir = venv_dir
@ -219,8 +240,7 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin):
# Let's remove the pip binary
pip_bin = os.path.join(venv_dir, 'bin', 'pip')
py_dir = 'python{0}.{1}'.format(*sys.version_info[:2])
site_dir = os.path.join(venv_dir, 'lib', py_dir, 'site-packages')
site_dir = self.run_function('virtualenv.get_distribution_path', [venv_dir, 'pip'])
if salt.utils.platform.is_windows():
pip_bin = os.path.join(venv_dir, 'Scripts', 'pip.exe')
site_dir = os.path.join(venv_dir, 'lib', 'site-packages')
@ -274,6 +294,7 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin):
@destructiveTest
@skip_if_not_root
@skipIf(not CAN_RUNAS, 'Runas support required')
@with_system_user('issue-6912', on_existing='delete', delete=True,
password='PassWord1!')
@with_tempdir()
@ -317,6 +338,7 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin):
@destructiveTest
@skip_if_not_root
@skipIf(not CAN_RUNAS, 'Runas support required')
@with_system_user('issue-6912', on_existing='delete', delete=True,
password='PassWord1!')
@with_tempdir()

5
tests/unit/modules/test_dockermod.py
View file

@ -45,11 +45,8 @@ class DockerTestCase(TestCase, LoaderModuleMockMixin):
def setup_loader_modules(self):
utils = salt.loader.utils(
salt.config.DEFAULT_MINION_OPTS,
whitelist=['state']
whitelist=['args', 'docker', 'json', 'state', 'thin']
)
# Force the LazyDict to populate its references. Otherwise the lookup
# will fail inside the unit tests.
list(utils)
return {docker_mod: {'__context__': {'docker.docker_version': ''},
'__utils__': utils}}

48
tests/unit/output/test_table_out.py Normal file
View file

@ -0,0 +1,48 @@
# -*- coding: utf-8 -*-
'''
unittests for table outputter
'''
# Import Python Libs
from __future__ import absolute_import, print_function, unicode_literals
# Import Salt Testing Libs
from tests.support.mixins import LoaderModuleMockMixin
from tests.support.unit import TestCase
# Import Salt Libs
import salt.output.table_out as table_out
import salt.utils.stringutils
class TableTestCase(TestCase, LoaderModuleMockMixin):
'''
Test cases for salt.output.table_out
'''
def setup_loader_modules(self):
return {table_out: {}}
# The test data should include unicode chars, and in Python 2 there should
# be an example both of an encoded str type and an actual unicode type.
# Since unicode_literals is imported, we will achieve the former using
# salt.utils.stringutils.to_str and the latter by simply using a string
# literal.
data = [
{'Food': salt.utils.stringutils.to_str('яйца, бекон, колбаса и спам'),
'Price': 5.99},
{'Food': 'спам, спам, спам, яйца и спам',
'Price': 3.99},
]
def test_output(self):
ret = table_out.output(self.data)
self.assertEqual(
ret,
(' -----------------------------------------\n'
' | Food | Price |\n'
' -----------------------------------------\n'
' | яйца, бекон, колбаса и спам | 5.99 |\n'
' -----------------------------------------\n'
' | спам, спам, спам, яйца и спам | 3.99 |\n'
' -----------------------------------------')
)

5
tests/unit/states/test_boto_cloudfront.py
View file

@ -26,12 +26,9 @@ class BotoCloudfrontTestCase(TestCase, LoaderModuleMockMixin):
def setup_loader_modules(self):
utils = salt.loader.utils(
self.opts,
whitelist=['boto3', 'dictdiffer', 'yamldumper'],
whitelist=['boto3', 'dictdiffer', 'yaml'],
context={},
)
# Force the LazyDict to populate its references. Otherwise the lookup
# will fail inside the unit tests.
list(utils)
return {
boto_cloudfront: {
'__utils__': utils,

5
tests/unit/states/test_boto_sqs.py
View file

@ -25,12 +25,9 @@ class BotoSqsTestCase(TestCase, LoaderModuleMockMixin):
def setup_loader_modules(self):
utils = salt.loader.utils(
self.opts,
whitelist=['boto3', 'yamldumper'],
whitelist=['boto3', 'yaml'],
context={}
)
# Force the LazyDict to populate its references. Otherwise the lookup
# will fail inside the unit tests.
list(utils)
return {
boto_sqs: {
'__utils__': utils,