Merge branch '2017.7' into 2017.7

This commit is contained in:
Mike Place 2017-11-20 19:07:03 +00:00 committed by GitHub
commit 98536110d9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 281 additions and 93 deletions

View file

@ -376,6 +376,22 @@ The above example will force the minion to use the :py:mod:`systemd
.. __: https://github.com/saltstack/salt/issues/new
Logging Restrictions
--------------------
As a rule, logging should not be done anywhere in a Salt module before it is
loaded. This rule apples to all code that would run before the ``__virtual__()``
function, as well as the code within the ``__virtual__()`` function itself.
If logging statements are made before the virtual function determines if
the module should be loaded, then those logging statements will be called
repeatedly. This clutters up log files unnecessarily.
Exceptions may be considered for logging statements made at the ``trace`` level.
However, it is better to provide the necessary information by another means.
One method is to :ref:`return error information <modules-error-info>` in the
``__virtual__()`` function.
.. _modules-virtual-name:
``__virtualname__``

View file

@ -160,6 +160,7 @@ class Master(parsers.MasterOptionParser, DaemonsMixin): # pylint: disable=no-in
self.config['user'],
permissive=self.config['permissive_pki_access'],
pki_dir=self.config['pki_dir'],
root_dir=self.config['root_dir'],
)
# Clear out syndics from cachedir
for syndic_file in os.listdir(self.config['syndic_dir']):
@ -280,6 +281,7 @@ class Minion(parsers.MinionOptionParser, DaemonsMixin): # pylint: disable=no-in
self.config['user'],
permissive=self.config['permissive_pki_access'],
pki_dir=self.config['pki_dir'],
root_dir=self.config['root_dir'],
)
except OSError as error:
self.environment_failure(error)
@ -467,6 +469,7 @@ class ProxyMinion(parsers.ProxyMinionOptionParser, DaemonsMixin): # pylint: dis
self.config['user'],
permissive=self.config['permissive_pki_access'],
pki_dir=self.config['pki_dir'],
root_dir=self.config['root_dir'],
)
except OSError as error:
self.environment_failure(error)
@ -575,6 +578,7 @@ class Syndic(parsers.SyndicOptionParser, DaemonsMixin): # pylint: disable=no-in
self.config['user'],
permissive=self.config['permissive_pki_access'],
pki_dir=self.config['pki_dir'],
root_dir=self.config['root_dir'],
)
except OSError as error:
self.environment_failure(error)

View file

@ -32,7 +32,10 @@ class SPM(parsers.SPMParser):
v_dirs = [
self.config['cachedir'],
]
verify_env(v_dirs, self.config['user'],)
verify_env(v_dirs,
self.config['user'],
root_dir=self.config['root_dir'],
)
verify_log(self.config)
client = salt.spm.SPMClient(ui, self.config)
client.run(self.args)

View file

@ -903,6 +903,8 @@ class Single(object):
ret = json.dumps({'local': opts_pkg})
return ret, retcode
if 'known_hosts_file' in self.opts:
opts_pkg['known_hosts_file'] = self.opts['known_hosts_file']
opts_pkg['file_roots'] = self.opts['file_roots']
opts_pkg['pillar_roots'] = self.opts['pillar_roots']
opts_pkg['ext_pillar'] = self.opts['ext_pillar']

View file

@ -66,7 +66,8 @@ class SaltCloud(parsers.SaltCloudParser):
if self.config['verify_env']:
verify_env(
[os.path.dirname(self.config['conf_file'])],
salt_master_user
salt_master_user,
root_dir=self.config['root_dir'],
)
logfile = self.config['log_file']
if logfile is not None and not logfile.startswith('tcp://') \

View file

@ -6,9 +6,10 @@ Module for sending messages to Mattermost
:configuration: This module can be used by either passing an api_url and hook
directly or by specifying both in a configuration profile in the salt
master/minion config.
For example:
master/minion config. For example:
.. code-block:: yaml
mattermost:
hook: peWcBiMOS9HrZG15peWcBiMOS9HrZG15
api_url: https://example.com
@ -35,6 +36,7 @@ __virtualname__ = 'mattermost'
def __virtual__():
'''
Return virtual name of the module.
:return: The virtual name of the module.
'''
return __virtualname__
@ -43,6 +45,7 @@ def __virtual__():
def _get_hook():
'''
Retrieves and return the Mattermost's configured hook
:return: String: the hook string
'''
hook = __salt__['config.get']('mattermost.hook') or \
@ -56,6 +59,7 @@ def _get_hook():
def _get_api_url():
'''
Retrieves and return the Mattermost's configured api url
:return: String: the api url string
'''
api_url = __salt__['config.get']('mattermost.api_url') or \
@ -69,6 +73,7 @@ def _get_api_url():
def _get_channel():
'''
Retrieves the Mattermost's configured channel
:return: String: the channel string
'''
channel = __salt__['config.get']('mattermost.channel') or \
@ -80,6 +85,7 @@ def _get_channel():
def _get_username():
'''
Retrieves the Mattermost's configured username
:return: String: the username string
'''
username = __salt__['config.get']('mattermost.username') or \
@ -95,14 +101,18 @@ def post_message(message,
hook=None):
'''
Send a message to a Mattermost channel.
:param channel: The channel name, either will work.
:param username: The username of the poster.
:param message: The message to send to the Mattermost channel.
:param api_url: The Mattermost api url, if not specified in the configuration.
:param hook: The Mattermost hook, if not specified in the configuration.
:return: Boolean if message was sent successfully.
CLI Example:
.. code-block:: bash
salt '*' mattermost.post_message message='Build is done"
'''
if not api_url:

View file

@ -52,6 +52,7 @@ from salt.exceptions import (CommandExecutionError,
SaltRenderError)
import salt.utils
import salt.utils.pkg
import salt.utils.path
import salt.syspaths
import salt.payload
from salt.exceptions import MinionError
@ -641,33 +642,10 @@ def _get_repo_details(saltenv):
# Do some safety checks on the repo_path as its contents can be removed,
# this includes check for bad coding
system_root = os.environ.get('SystemRoot', r'C:\Windows')
deny_paths = (
r'[a-z]\:\\$', # C:\, D:\, etc
r'\\$', # \
re.escape(system_root) # C:\Windows
)
if not salt.utils.path.safe_path(
path=local_dest,
allow_path='\\'.join([system_root, 'TEMP'])):
# Since the above checks anything in C:\Windows, there are some
# directories we may want to make exceptions for
allow_paths = (
re.escape('\\'.join([system_root, 'TEMP'])), # C:\Windows\TEMP
)
# Check the local_dest to make sure it's not one of the bad paths
good_path = True
for d_path in deny_paths:
if re.match(d_path, local_dest, flags=re.IGNORECASE) is not None:
# Found deny path
good_path = False
# If local_dest is one of the bad paths, check for exceptions
if not good_path:
for a_path in allow_paths:
if re.match(a_path, local_dest, flags=re.IGNORECASE) is not None:
# Found exception
good_path = True
if not good_path:
raise CommandExecutionError(
'Attempting to delete files from a possibly unsafe location: '
'{0}'.format(local_dest)

View file

@ -1496,13 +1496,8 @@ def accept_vpc_peering_connection(name=None, conn_id=None, conn_name=None,
'''
log.debug('Called state to accept VPC peering connection')
pending = __salt__['boto_vpc.is_peering_connection_pending'](
conn_id=conn_id,
conn_name=conn_name,
region=region,
key=key,
keyid=keyid,
profile=profile
)
conn_id=conn_id, conn_name=conn_name, region=region, key=key,
keyid=keyid, profile=profile)
ret = {
'name': name,
@ -1511,32 +1506,27 @@ def accept_vpc_peering_connection(name=None, conn_id=None, conn_name=None,
'comment': 'Boto VPC peering state'
}
if not pending['exists']:
if not pending:
ret['result'] = True
ret['changes'].update({
'old': 'No pending VPC peering connection found. '
'Nothing to be done.'
})
ret['changes'].update({'old':
'No pending VPC peering connection found. Nothing to be done.'})
return ret
if __opts__['test']:
ret['changes'].update({'old': 'Pending VPC peering connection found '
'and can be accepted'})
ret['changes'].update({'old':
'Pending VPC peering connection found and can be accepted'})
return ret
log.debug('Calling module to accept this VPC peering connection')
result = __salt__['boto_vpc.accept_vpc_peering_connection'](
conn_id=conn_id, name=conn_name, region=region, key=key,
fun = 'boto_vpc.accept_vpc_peering_connection'
log.debug('Calling `{0}()` to accept this VPC peering connection'.format(fun))
result = __salt__[fun](conn_id=conn_id, name=conn_name, region=region, key=key,
keyid=keyid, profile=profile)
if 'error' in result:
ret['comment'] = "Failed to request VPC peering: {0}".format(result['error'])
ret['comment'] = "Failed to accept VPC peering: {0}".format(result['error'])
ret['result'] = False
return ret
ret['changes'].update({
'old': '',
'new': result['msg']
})
ret['changes'].update({'old': '', 'new': result['msg']})
return ret

View file

@ -2016,7 +2016,11 @@ def check_state_result(running, recurse=False, highstate=None):
ret = True
for state_id, state_result in six.iteritems(running):
if not recurse and not isinstance(state_result, dict):
expected_type = dict
# The __extend__ state is a list
if "__extend__" == state_id:
expected_type = list
if not recurse and not isinstance(state_result, expected_type):
ret = False
if ret and isinstance(state_result, dict):
result = state_result.get('result', _empty)

View file

@ -15,18 +15,19 @@ Utils for the NAPALM modules and proxy.
.. versionadded:: 2017.7.0
'''
# Import Python libs
from __future__ import absolute_import
import traceback
import logging
import importlib
from functools import wraps
log = logging.getLogger(__file__)
import salt.utils
# Import Salt libs
from salt.ext import six as six
import salt.output
import salt.utils
# Import third party lib
# Import third party libs
try:
# will try to import NAPALM
# https://github.com/napalm-automation/napalm
@ -36,21 +37,16 @@ try:
# pylint: enable=W0611
HAS_NAPALM = True
HAS_NAPALM_BASE = False # doesn't matter anymore, but needed for the logic below
log.debug('napalm seems to be installed')
try:
NAPALM_MAJOR = int(napalm.__version__.split('.')[0])
log.debug('napalm version: %s', napalm.__version__)
except AttributeError:
NAPALM_MAJOR = 0
except ImportError:
log.info('napalm doesnt seem to be installed, trying to import napalm_base')
HAS_NAPALM = False
try:
import napalm_base
log.debug('napalm_base seems to be installed')
HAS_NAPALM_BASE = True
except ImportError:
log.info('napalm_base doesnt seem to be installed either')
HAS_NAPALM_BASE = False
try:
@ -62,7 +58,7 @@ try:
except ImportError:
HAS_CONN_CLOSED_EXC_CLASS = False
from salt.ext import six as six
log = logging.getLogger(__file__)
def is_proxy(opts):
@ -98,8 +94,6 @@ def virtual(opts, virtualname, filename):
Returns the __virtual__.
'''
if ((HAS_NAPALM and NAPALM_MAJOR >= 2) or HAS_NAPALM_BASE) and (is_proxy(opts) or is_minion(opts)):
if HAS_NAPALM_BASE:
log.info('You still seem to use napalm_base. Please consider upgrading to napalm >= 2.0.0')
return virtualname
else:
return (

View file

@ -191,6 +191,16 @@ def get_entry_multi(dict_, pairs, raise_error=True):
return {}
def get_endpoint_url_v3(catalog, service_type, region_name):
for service_entry in catalog:
if service_entry['type'] == service_type:
for endpoint_entry in service_entry['endpoints']:
if (endpoint_entry['region'] == region_name and
endpoint_entry['interface'] == 'public'):
return endpoint_entry['url']
return None
def sanatize_novaclient(kwargs):
variables = (
'username', 'api_key', 'project_id', 'auth_url', 'insecure',
@ -353,21 +363,16 @@ class SaltNova(object):
def _v3_setup(self, region_name):
if region_name is not None:
servers_endpoints = get_entry(self.catalog, 'type', 'compute')['endpoints']
self.kwargs['bypass_url'] = get_entry_multi(
servers_endpoints,
[('region', region_name), ('interface', 'public')]
)['url']
self.client_kwargs['bypass_url'] = get_endpoint_url_v3(self.catalog, 'compute', region_name)
log.debug('Using Nova bypass_url: %s', self.client_kwargs['bypass_url'])
self.compute_conn = client.Client(version=self.version, session=self.session, **self.client_kwargs)
volume_endpoints = get_entry(self.catalog, 'type', 'volume', raise_error=False).get('endpoints', {})
if volume_endpoints:
if region_name is not None:
self.kwargs['bypass_url'] = get_entry_multi(
volume_endpoints,
[('region', region_name), ('interface', 'public')]
)['url']
self.client_kwargs['bypass_url'] = get_endpoint_url_v3(self.catalog, 'volume', region_name)
log.debug('Using Cinder bypass_url: %s', self.client_kwargs['bypass_url'])
self.volume_conn = client.Client(version=self.version, session=self.session, **self.client_kwargs)
if hasattr(self, 'extensions'):

View file

@ -174,3 +174,60 @@ def _get_reparse_data(path):
win32file.CloseHandle(fileHandle)
return reparseData
def safe_path(path, allow_path=None):
r'''
.. versionadded:: 2017.7.3
Checks that the path is safe for modification by Salt. For example, you
wouldn't want to have salt delete the contents of ``C:\Windows``. The
following directories are considered unsafe:
- C:\, D:\, E:\, etc.
- \
- C:\Windows
Args:
path (str): The path to check
allow_paths (str, list): A directory or list of directories inside of
path that may be safe. For example: ``C:\Windows\TEMP``
Returns:
bool: True if safe, otherwise False
'''
# Create regex definitions for directories that may be unsafe to modify
system_root = os.environ.get('SystemRoot', 'C:\\Windows')
deny_paths = (
r'[a-z]\:\\$', # C:\, D:\, etc
r'\\$', # \
re.escape(system_root) # C:\Windows
)
# Make allow_path a list
if allow_path and not isinstance(allow_path, list):
allow_path = [allow_path]
# Create regex definition for directories we may want to make exceptions for
allow_paths = list()
if allow_path:
for item in allow_path:
allow_paths.append(re.escape(item))
# Check the path to make sure it's not one of the bad paths
good_path = True
for d_path in deny_paths:
if re.match(d_path, path, flags=re.IGNORECASE) is not None:
# Found deny path
good_path = False
# If local_dest is one of the bad paths, check for exceptions
if not good_path:
for a_path in allow_paths:
if re.match(a_path, path, flags=re.IGNORECASE) is not None:
# Found exception
good_path = True
return good_path

View file

@ -31,6 +31,8 @@ import salt.utils
log = logging.getLogger(__name__)
ROOT_DIR = 'c:\\salt' if salt.utils.is_windows() else '/'
def zmq_version():
'''
@ -192,13 +194,13 @@ def verify_files(files, user):
return True
def verify_env(dirs, user, permissive=False, pki_dir='', skip_extra=False):
def verify_env(dirs, user, permissive=False, pki_dir='', skip_extra=False, root_dir=ROOT_DIR):
'''
Verify that the named directories are in place and that the environment
can shake the salt
'''
if salt.utils.is_windows():
return win_verify_env(dirs, permissive, pki_dir, skip_extra)
return win_verify_env(root_dir, dirs, permissive, pki_dir, skip_extra)
import pwd # after confirming not running Windows
try:
pwnam = pwd.getpwnam(user)
@ -523,18 +525,21 @@ def verify_log(opts):
log.warning('Insecure logging configuration detected! Sensitive data may be logged.')
def win_verify_env(dirs, permissive=False, pki_dir='', skip_extra=False):
def win_verify_env(path, dirs, permissive=False, pki_dir='', skip_extra=False):
'''
Verify that the named directories are in place and that the environment
can shake the salt
'''
import salt.utils.win_functions
import salt.utils.win_dacl
import salt.utils.path
# Get the root path directory where salt is installed
path = dirs[0]
while os.path.basename(path) not in ['salt', 'salt-tests-tmpdir']:
path, base = os.path.split(path)
# Make sure the file_roots is not set to something unsafe since permissions
# on that directory are reset
if not salt.utils.path.safe_path(path=path):
raise CommandExecutionError(
'`file_roots` set to a possibly unsafe location: {0}'.format(path)
)
# Create the root path directory if missing
if not os.path.isdir(path):

View file

@ -983,7 +983,9 @@ class TestDaemon(object):
RUNTIME_VARS.TMP_PRODENV_STATE_TREE,
TMP,
],
RUNTIME_VARS.RUNNING_TESTS_USER)
RUNTIME_VARS.RUNNING_TESTS_USER,
root_dir=master_opts['root_dir'],
)
cls.master_opts = master_opts
cls.minion_opts = minion_opts

View file

@ -7,8 +7,13 @@ import shutil
# Import Salt Testing Libs
from tests.support.case import SSHCase
from tests.support.paths import TMP
# Import Salt Libs
from salt.ext import six
SSH_SLS = 'ssh_state_tests'
SSH_SLS_FILE = '/tmp/test'
class SSHStateTest(SSHCase):
@ -37,6 +42,87 @@ class SSHStateTest(SSHCase):
check_file = self.run_function('file.file_exists', ['/tmp/test'])
self.assertTrue(check_file)
def test_state_show_sls(self):
'''
test state.show_sls with salt-ssh
'''
ret = self.run_function('state.show_sls', [SSH_SLS])
self._check_dict_ret(ret=ret, val='__sls__', exp_ret=SSH_SLS)
check_file = self.run_function('file.file_exists', [SSH_SLS_FILE], wipe=False)
self.assertFalse(check_file)
def test_state_show_top(self):
'''
test state.show_top with salt-ssh
'''
ret = self.run_function('state.show_top')
self.assertEqual(ret, {u'base': [u'master_tops_test', u'core']})
def test_state_single(self):
'''
state.single with salt-ssh
'''
ret_out = {'name': 'itworked',
'result': True,
'comment': 'Success!'}
single = self.run_function('state.single',
['test.succeed_with_changes name=itworked'])
for key, value in six.iteritems(single):
self.assertEqual(value['name'], ret_out['name'])
self.assertEqual(value['result'], ret_out['result'])
self.assertEqual(value['comment'], ret_out['comment'])
def test_show_highstate(self):
'''
state.show_highstate with salt-ssh
'''
high = self.run_function('state.show_highstate')
destpath = os.path.join(TMP, 'testfile')
self.assertTrue(isinstance(high, dict))
self.assertTrue(destpath in high)
self.assertEqual(high[destpath]['__env__'], 'base')
def test_state_high(self):
'''
state.high with salt-ssh
'''
ret_out = {'name': 'itworked',
'result': True,
'comment': 'Success!'}
high = self.run_function('state.high', ['"{"itworked": {"test": ["succeed_with_changes"]}}"'])
for key, value in six.iteritems(high):
self.assertEqual(value['name'], ret_out['name'])
self.assertEqual(value['result'], ret_out['result'])
self.assertEqual(value['comment'], ret_out['comment'])
def test_show_lowstate(self):
'''
state.show_lowstate with salt-ssh
'''
low = self.run_function('state.show_lowstate')
self.assertTrue(isinstance(low, list))
self.assertTrue(isinstance(low[0], dict))
def test_state_low(self):
'''
state.low with salt-ssh
'''
ret_out = {'name': 'itworked',
'result': True,
'comment': 'Success!'}
low = self.run_function('state.low', ['"{"state": "test", "fun": "succeed_with_changes", "name": "itworked"}"'])
for key, value in six.iteritems(low):
self.assertEqual(value['name'], ret_out['name'])
self.assertEqual(value['result'], ret_out['result'])
self.assertEqual(value['comment'], ret_out['comment'])
def test_state_request_check_clear(self):
'''
test state.request system with salt-ssh
@ -60,7 +146,7 @@ class SSHStateTest(SSHCase):
run = self.run_function('state.run_request', wipe=False)
check_file = self.run_function('file.file_exists', ['/tmp/test'], wipe=False)
check_file = self.run_function('file.file_exists', [SSH_SLS_FILE], wipe=False)
self.assertTrue(check_file)
def tearDown(self):
@ -70,3 +156,6 @@ class SSHStateTest(SSHCase):
salt_dir = self.run_function('config.get', ['thin_dir'], wipe=False)
if os.path.exists(salt_dir):
shutil.rmtree(salt_dir)
if os.path.exists(SSH_SLS_FILE):
os.remove(SSH_SLS_FILE)

View file

@ -7,7 +7,6 @@ from __future__ import absolute_import
# Import Salt Testing libs
from tests.support.case import ModuleCase
from tests.support.unit import skipIf
from tests.support.helpers import destructiveTest
from tests.support.mixins import SaltReturnAssertsMixin
@ -15,32 +14,58 @@ from tests.support.mixins import SaltReturnAssertsMixin
import salt.utils
INIT_DELAY = 5
SERVICE_NAME = 'crond'
@destructiveTest
@skipIf(salt.utils.which('crond') is None, 'crond not installed')
class ServiceTest(ModuleCase, SaltReturnAssertsMixin):
'''
Validate the service state
'''
def setUp(self):
self.service_name = 'cron'
cmd_name = 'crontab'
os_family = self.run_function('grains.get', ['os_family'])
if os_family == 'RedHat':
self.service_name = 'crond'
elif os_family == 'Arch':
self.service_name = 'systemd-journald'
cmd_name = 'systemctl'
if salt.utils.which(cmd_name) is None:
self.skipTest('{0} is not installed'.format(cmd_name))
def check_service_status(self, exp_return):
'''
helper method to check status of service
'''
check_status = self.run_function('service.status', name=SERVICE_NAME)
check_status = self.run_function('service.status',
name=self.service_name)
if check_status is not exp_return:
self.fail('status of service is not returning correctly')
def test_service_running(self):
'''
test service.running state module
'''
stop_service = self.run_function('service.stop', self.service_name)
self.assertTrue(stop_service)
self.check_service_status(False)
start_service = self.run_state('service.running',
name=self.service_name)
self.assertTrue(start_service)
self.check_service_status(True)
def test_service_dead(self):
'''
test service.dead state module
'''
start_service = self.run_state('service.running', name=SERVICE_NAME)
start_service = self.run_state('service.running',
name=self.service_name)
self.assertSaltTrueReturn(start_service)
self.check_service_status(True)
ret = self.run_state('service.dead', name=SERVICE_NAME)
ret = self.run_state('service.dead', name=self.service_name)
self.assertSaltTrueReturn(ret)
self.check_service_status(False)
@ -48,11 +73,12 @@ class ServiceTest(ModuleCase, SaltReturnAssertsMixin):
'''
test service.dead state module with init_delay arg
'''
start_service = self.run_state('service.running', name=SERVICE_NAME)
start_service = self.run_state('service.running',
name=self.service_name)
self.assertSaltTrueReturn(start_service)
self.check_service_status(True)
ret = self.run_state('service.dead', name=SERVICE_NAME,
ret = self.run_state('service.dead', name=self.service_name,
init_delay=INIT_DELAY)
self.assertSaltTrueReturn(ret)
self.check_service_status(False)

View file

@ -107,7 +107,9 @@ class AdaptedConfigurationTestCaseMixin(object):
rdict['sock_dir'],
conf_dir
],
RUNTIME_VARS.RUNNING_TESTS_USER)
RUNTIME_VARS.RUNNING_TESTS_USER,
root_dir=rdict['root_dir'],
)
rdict['config_dir'] = conf_dir
rdict['conf_file'] = os.path.join(conf_dir, config_for)

View file

@ -111,7 +111,7 @@ class TestVerify(TestCase):
def test_verify_env(self):
root_dir = tempfile.mkdtemp(dir=TMP)
var_dir = os.path.join(root_dir, 'var', 'log', 'salt')
verify_env([var_dir], getpass.getuser())
verify_env([var_dir], getpass.getuser(), root_dir=root_dir)
self.assertTrue(os.path.exists(var_dir))
dir_stat = os.stat(var_dir)
self.assertEqual(dir_stat.st_uid, os.getuid())