saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #47564 from rallytime/merge-2018.3

Browse source 
[2018.3] Merge forward from 2017.7 to 2018.3
This commit is contained in:
Nicole Thomas 2018-05-10 15:13:35 -04:00 • committed by GitHub
commit 9836a7e62b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 173 additions and 113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
doc/topics/tutorials/gitfs.rst
View file

@ -699,15 +699,24 @@ repository to be served up from the Salt fileserver path
Mountpoints can also be configured on a :ref:`per-remote basis
<gitfs-per-remote-config>`.
Using gitfs in Masterless Mode
==============================
Since 2014.7.0, gitfs can be used in masterless mode. To do so, simply add the
gitfs configuration parameters (and set :conf_master:`fileserver_backend`) in
the _minion_ config file instead of the master config file.
Using gitfs Alongside Other Backends
====================================
Sometimes it may make sense to use multiple backends; for instance, if ``sls``
files are stored in git but larger files are stored directly on the master.
The cascading lookup logic used for multiple remotes is also used with
multiple backends. If the ``fileserver_backend`` option contains
multiple backends:
The cascading lookup logic used for multiple remotes is also used with multiple
backends. If the :conf_master:`fileserver_backend` option contains multiple
backends:
.. code-block:: yaml
@ -719,7 +728,6 @@ Then the ``roots`` backend (the default backend of files in ``/srv/salt``) will
be searched first for the requested file; then, if it is not found on the
master, each configured git remote will be searched.
Branches, Environments, and Top Files
=====================================

4
salt/modules/boto3_route53.py
View file

@ -551,6 +551,10 @@ def associate_vpc_with_hosted_zone(HostedZoneId=None, Name=None, VPCId=None,
r = conn.associate_vpc_with_hosted_zone(**args)
return _wait_for_sync(r['ChangeInfo']['Id'], conn)
except ClientError as e:
if e.response.get('Error', {}).get('Code') == 'ConflictingDomainExists':
log.debug('VPC Association already exists.')
# return True since the current state is the desired one
return True
if tries and e.response.get('Error', {}).get('Code') == 'Throttling':
log.debug('Throttled by AWS API.')
time.sleep(3)

8
salt/modules/cmdmod.py
View file

@ -478,10 +478,18 @@ def _run(cmd,
for k, v in six.iteritems(env_runas)
)
env_runas.update(env)
# Fix platforms like Solaris that don't set a USER env var in the
# user's default environment as obtained above.
if env_runas.get('USER') != runas:
env_runas['USER'] = runas
# Fix some corner cases where shelling out to get the user's
# environment returns the wrong home directory.
runas_home = os.path.expanduser('~{0}'.format(runas))
if env_runas.get('HOME') != runas_home:
env_runas['HOME'] = runas_home
env = env_runas
except ValueError as exc:
log.exception('Error raised retrieving environment for user %s', runas)

97
salt/modules/pip.py
View file

@ -460,13 +460,10 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914
Path to requirements
bin_env
Path to pip bin or path to virtualenv. If doing a system install,
and want to use a specific pip bin (pip-2.7, pip-2.6, etc..) just
specify the pip bin you want.
.. note::
If installing into a virtualenv, just use the path to the
virtualenv (e.g. ``/home/code/path/to/virtualenv/``)
Path to pip (or to a virtualenv). This can be used to specify the path
to the pip to use when more than one Python release is installed (e.g.
``/usr/bin/pip-2.7`` or ``/usr/bin/pip-2.6``. If a directory path is
specified, it is assumed to be a virtualenv.
use_wheel
Prefer wheel archives (requires pip>=1.4)
@ -569,7 +566,7 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914
The user under which to run pip
cwd
Current working directory to run pip from
Directory from which to run pip
pre_releases
Include pre-releases in the available versions
@ -941,36 +938,38 @@ def uninstall(pkgs=None,
saltenv='base',
use_vt=False):
'''
Uninstall packages with pip
Uninstall packages individually or from a pip requirements file. Uninstall
packages globally or from a virtualenv.
Uninstall packages individually or from a pip requirements file
pkgs
comma separated list of packages to install
requirements
path to requirements.
Path to requirements file
bin_env
path to pip bin or path to virtualenv. If doing an uninstall from
the system python and want to use a specific pip bin (pip-2.7,
pip-2.6, etc..) just specify the pip bin you want.
If uninstalling from a virtualenv, just use the path to the virtualenv
(/home/code/path/to/virtualenv/)
Path to pip (or to a virtualenv). This can be used to specify the path
to the pip to use when more than one Python release is installed (e.g.
``/usr/bin/pip-2.7`` or ``/usr/bin/pip-2.6``. If a directory path is
specified, it is assumed to be a virtualenv.
log
Log file where a complete (maximum verbosity) record will be kept
proxy
Specify a proxy in the form
user:passwd@proxy.server:port. Note that the
user:password@ is optional and required only if you
are behind an authenticated proxy. If you provide
user@proxy.server:port then you will be prompted for a
password.
Specify a proxy in the format ``user:passwd@proxy.server:port``. Note
that the ``user:password@`` is optional and required only if you are
behind an authenticated proxy. If you provide
``user@proxy.server:port`` then you will be prompted for a password.
timeout
Set the socket timeout (default 15 seconds)
user
The user under which to run pip
cwd
Current working directory to run pip from
Directory from which to run pip
use_vt
Use VT terminal emulation (see output while installing)
@ -982,7 +981,6 @@ def uninstall(pkgs=None,
salt '*' pip.uninstall requirements=/path/to/requirements.txt
salt '*' pip.uninstall <package name> bin_env=/path/to/virtualenv
salt '*' pip.uninstall <package name> bin_env=/path/to/pip_bin
'''
cmd = _get_pip_bin(bin_env)
cmd.extend(['uninstall', '-y'])
@ -1065,32 +1063,27 @@ def freeze(bin_env=None,
virtualenv
bin_env
path to pip bin or path to virtualenv. If doing an uninstall from
the system python and want to use a specific pip bin (pip-2.7,
pip-2.6, etc..) just specify the pip bin you want.
If uninstalling from a virtualenv, just use the path to the virtualenv
(/home/code/path/to/virtualenv/)
Path to pip (or to a virtualenv). This can be used to specify the path
to the pip to use when more than one Python release is installed (e.g.
``/usr/bin/pip-2.7`` or ``/usr/bin/pip-2.6``. If a directory path is
specified, it is assumed to be a virtualenv.
user
The user under which to run pip
cwd
Current working directory to run pip from
Directory from which to run pip
.. note::
If the version of pip available is older than 8.0.3, the list will not
include the packages pip, wheel, setuptools, or distribute even if they
are installed.
include the packages ``pip``, ``wheel``, ``setuptools``, or
``distribute`` even if they are installed.
CLI Example:
.. code-block:: bash
salt '*' pip.freeze /home/code/path/to/virtualenv/
.. versionchanged:: 2016.11.2
The packages pip, wheel, setuptools, and distribute are included if the
installed pip is new enough.
salt '*' pip.freeze bin_env=/home/code/path/to/virtualenv
'''
cmd = _get_pip_bin(bin_env)
cmd.append('freeze')
@ -1135,21 +1128,16 @@ def list_(prefix=None,
.. note::
If the version of pip available is older than 8.0.3, the packages
wheel, setuptools, and distribute will not be reported by this function
even if they are installed. Unlike
:py:func:`pip.freeze <salt.modules.pip.freeze>`, this function always
reports the version of pip which is installed.
``wheel``, ``setuptools``, and ``distribute`` will not be reported by
this function even if they are installed. Unlike :py:func:`pip.freeze
<salt.modules.pip.freeze>`, this function always reports the version of
pip which is installed.
CLI Example:
.. code-block:: bash
salt '*' pip.list salt
.. versionchanged:: 2016.11.2
The packages wheel, setuptools, and distribute are included if the
installed pip is new enough.
'''
packages = {}
@ -1457,9 +1445,10 @@ def list_all_versions(pkg,
The package to check
bin_env
Path to pip bin or path to virtualenv. If doing a system install,
and want to use a specific pip bin (pip-2.7, pip-2.6, etc..) just
specify the pip bin you want.
Path to pip (or to a virtualenv). This can be used to specify the path
to the pip to use when more than one Python release is installed (e.g.
``/usr/bin/pip-2.7`` or ``/usr/bin/pip-2.6``. If a directory path is
specified, it is assumed to be a virtualenv.
include_alpha
Include alpha versions in the list
@ -1474,7 +1463,7 @@ def list_all_versions(pkg,
The user under which to run pip
cwd
Current working directory to run pip from
Directory from which to run pip
CLI Example:

2
salt/modules/status.py
View file

@ -989,7 +989,7 @@ def diskusage(*args):
elif __grains__['kernel'] in ('FreeBSD', 'SunOS'):
ifile = __salt__['cmd.run']('mount -p').splitlines()
else:
ifile = []
raise CommandExecutionError('status.diskusage not yet supported on this platform')
for line in ifile:
comps = line.split()

64
salt/state.py
View file

@ -3430,43 +3430,45 @@ class BaseHighState(object):
'Specified SLS {0} on local filesystem cannot '
'be found.'.format(sls)
)
state = None
if not fn_:
errors.append(
'Specified SLS {0} in saltenv {1} is not '
'available on the salt master or through a configured '
'fileserver'.format(sls, saltenv)
)
state = None
try:
state = compile_template(fn_,
self.state.rend,
self.state.opts['renderer'],
self.state.opts['renderer_blacklist'],
self.state.opts['renderer_whitelist'],
saltenv,
sls,
rendered_sls=mods
)
except SaltRenderError as exc:
msg = 'Rendering SLS \'{0}:{1}\' failed: {2}'.format(
saltenv, sls, exc
)
log.critical(msg)
errors.append(msg)
except Exception as exc:
msg = 'Rendering SLS {0} failed, render error: {1}'.format(
sls, exc
)
log.critical(
msg,
# Show the traceback if the debug logging level is enabled
exc_info_on_loglevel=logging.DEBUG
)
errors.append('{0}\n{1}'.format(msg, traceback.format_exc()))
try:
mods.add('{0}:{1}'.format(saltenv, sls))
except AttributeError:
pass
else:
try:
state = compile_template(fn_,
self.state.rend,
self.state.opts['renderer'],
self.state.opts['renderer_blacklist'],
self.state.opts['renderer_whitelist'],
saltenv,
sls,
rendered_sls=mods
)
except SaltRenderError as exc:
msg = 'Rendering SLS \'{0}:{1}\' failed: {2}'.format(
saltenv, sls, exc
)
log.critical(msg)
errors.append(msg)
except Exception as exc:
msg = 'Rendering SLS {0} failed, render error: {1}'.format(
sls, exc
)
log.critical(
msg,
# Show the traceback if the debug logging level is enabled
exc_info_on_loglevel=logging.DEBUG
)
errors.append('{0}\n{1}'.format(msg, traceback.format_exc()))
try:
mods.add('{0}:{1}'.format(saltenv, sls))
except AttributeError:
pass
if state:
if not isinstance(state, dict):
errors.append(

4
tests/integration/modules/test_status.py
View file

@ -49,7 +49,9 @@ class StatusModuleTest(ModuleCase):
status.diskusage
'''
ret = self.run_function('status.diskusage')
if salt.utils.platform.is_windows():
if salt.utils.platform.is_darwin():
self.assertIn('not yet supported on this platform', ret)
elif salt.utils.platform.is_windows():
self.assertTrue(isinstance(ret['percent'], float))
else:
self.assertIn('total', str(ret))

6
tests/integration/states/test_pip.py → tests/integration/states/test_pip_state.py
View file

@ -3,7 +3,7 @@
:codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)`
tests.integration.states.pip
tests.integration.states.pip_state
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
@ -300,7 +300,7 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin):
# pip install passing the package name in `name`
ret = self.run_state(
'pip.installed', name='pep8', user=username, bin_env=venv_dir,
no_cache_dir=True, password='PassWord1!')
password='PassWord1!')
self.assertSaltTrueReturn(ret)
if HAS_PWD:
@ -350,7 +350,7 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin):
ret = self.run_state(
'pip.installed', name='', user=username, bin_env=venv_dir,
requirements='salt://issue-6912-requirements.txt',
no_cache_dir=True, password='PassWord1!')
password='PassWord1!')
self.assertSaltTrueReturn(ret)
if HAS_PWD:

31
tests/integration/utils/testprogram.py
View file

@ -20,6 +20,7 @@ import tempfile
import time
import salt.utils.files
import salt.utils.platform
import salt.utils.process
import salt.utils.psutil_compat as psutils
import salt.utils.yaml
@ -28,6 +29,7 @@ from salt.ext import six
from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin
from tests.support.unit import TestCase
from tests.support.helpers import win32_kill_process_tree
from tests.support.paths import CODE_DIR
from tests.support.processes import terminate_process, terminate_process_list
@ -413,9 +415,6 @@ class TestProgram(six.with_metaclass(TestProgramMeta, object)):
popen_kwargs['preexec_fn'] = detach_from_parent_group
elif sys.platform.lower().startswith('win') and timeout is not None:
raise RuntimeError('Timeout is not supported under windows')
self.argv = [self.program]
self.argv.extend(args)
log.debug('TestProgram.run: %s Environment %s', self.argv, env_delta)
@ -430,16 +429,26 @@ class TestProgram(six.with_metaclass(TestProgramMeta, object)):
if datetime.now() > stop_at:
if term_sent is False:
# Kill the process group since sending the term signal
# would only terminate the shell, not the command
# executed in the shell
os.killpg(os.getpgid(process.pid), signal.SIGINT)
term_sent = True
continue
if salt.utils.platform.is_windows():
_, alive = win32_kill_process_tree(process.pid)
if alive:
log.error("Child processes still alive: %s", alive)
else:
# Kill the process group since sending the term signal
# would only terminate the shell, not the command
# executed in the shell
os.killpg(os.getpgid(process.pid), signal.SIGINT)
term_sent = True
continue
try:
# As a last resort, kill the process group
os.killpg(os.getpgid(process.pid), signal.SIGKILL)
if salt.utils.platform.is_windows():
_, alive = win32_kill_process_tree(process.pid)
if alive:
log.error("Child processes still alive: %s", alive)
else:
# As a last resort, kill the process group
os.killpg(os.getpgid(process.pid), signal.SIGKILL)
process.wait()
except OSError as exc:
if exc.errno != errno.ESRCH:

3
tests/runtests.py
View file

@ -740,6 +740,9 @@ class SaltTestsuiteParser(SaltCoverageTestingParser):
with TestDaemon(self):
if self.options.name:
for name in self.options.name:
name = name.strip()
if not name:
continue
if os.path.isfile(name):
if not name.endswith('.py'):
continue

29
tests/support/case.py
View file

@ -29,13 +29,14 @@ from datetime import datetime, timedelta
# Import salt testing libs
from tests.support.unit import TestCase
from tests.support.helpers import RedirectStdStreams, requires_sshd_server
from tests.support.helpers import (
RedirectStdStreams, requires_sshd_server, win32_kill_process_tree
)
from tests.support.runtests import RUNTIME_VARS
from tests.support.mixins import AdaptedConfigurationTestCaseMixin, SaltClientTestCaseMixin
from tests.support.paths import ScriptPathMixin, INTEGRATION_TEST_DIR, CODE_DIR, PYEXEC, SCRIPT_DIR
# Import 3rd-party libs
import salt.utils.json
from salt.ext import six
from salt.ext.six.moves import cStringIO # pylint: disable=import-error
@ -287,11 +288,11 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
popen_kwargs['preexec_fn'] = detach_from_parent_group
elif sys.platform.lower().startswith('win') and timeout is not None:
raise RuntimeError('Timeout is not supported under windows')
process = subprocess.Popen(cmd, **popen_kwargs)
# Late import
import salt.utils.platform
if timeout is not None:
stop_at = datetime.now() + timedelta(seconds=timeout)
term_sent = False
@ -303,13 +304,23 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
# Kill the process group since sending the term signal
# would only terminate the shell, not the command
# executed in the shell
os.killpg(os.getpgid(process.pid), signal.SIGINT)
if salt.utils.platform.is_windows():
_, alive = win32_kill_process_tree(process.pid)
if alive:
log.error("Child processes still alive: %s", alive)
else:
os.killpg(os.getpgid(process.pid), signal.SIGINT)
term_sent = True
continue
try:
# As a last resort, kill the process group
os.killpg(os.getpgid(process.pid), signal.SIGKILL)
if salt.utils.platform.is_windows():
_, alive = win32_kill_process_tree(process.pid)
if alive:
log.error("Child processes still alive: %s", alive)
else:
os.killpg(os.getpgid(process.pid), signal.SIGINT)
except OSError as exc:
if exc.errno != errno.ESRCH:
# If errno is not "no such process", raise
@ -855,6 +866,10 @@ class SSHCase(ShellCase):
wipe=wipe, raw=raw)
log.debug('SSHCase run_function executed %s with arg %s', function, arg)
log.debug('SSHCase JSON return: %s', ret)
# Late import
import salt.utils.json
try:
return salt.utils.json.loads(ret)['localhost']
except Exception:

20
tests/support/helpers.py
View file

@ -1572,3 +1572,23 @@ class Webserver(object):
'''
self.ioloop.add_callback(self.ioloop.stop)
self.server_thread.join()
def win32_kill_process_tree(pid, sig=signal.SIGTERM, include_parent=True,
timeout=None, on_terminate=None):
'''
Kill a process tree (including grandchildren) with signal "sig" and return
a (gone, still_alive) tuple. "on_terminate", if specified, is a callabck
function which is called as soon as a child terminates.
'''
if pid == os.getpid():
raise RuntimeError("I refuse to kill myself")
parent = psutil.Process(pid)
children = parent.children(recursive=True)
if include_parent:
children.append(parent)
for p in children:
p.send_signal(sig)
gone, alive = psutil.wait_procs(children, timeout=timeout,
callback=on_terminate)
return (gone, alive)

0
tests/unit/states/test_pip.py → tests/unit/states/test_pip_state.py
View file

2
tests/whitelist.txt
View file

@ -37,7 +37,7 @@ integration.runners.test_jobs
integration.runners.test_salt
integration.sdb.test_env
integration.states.test_host
integration.states.test_pip
integration.states.test_pip_state
integration.states.test_reg
integration.states.test_renderers
integration.utils.testprogram