Merge branch '2019.2.1' into 2019.2.1_with_multimaster_bb_fixes

This commit is contained in:
Dmitry Kuzmenko 2019-08-22 12:54:46 +03:00
commit 9d280980a8
No known key found for this signature in database
GPG key ID: 4C7CAD30C95651DA
40 changed files with 485 additions and 363 deletions

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -140,7 +140,7 @@ wrappedNode('kitchen-slave', global_timeout, '#jenkins-prod-pr') {
withCredentials([[$class: 'StringBinding', credentialsId: 'codecov-upload-token-salt', variable: 'CODECOV_TOKEN']]) {
sh '''
if [ -n "${FORCE_FULL}" -a "${FORCE_FULL}" = "true" -a -f artifacts/coverage/coverage.xml ]; then
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}"
curl -L https://codecov.io/bash | /bin/sh -s -- -R $(pwd) -s artifacts/coverage/ -F "${CODECOV_FLAGS}" || true
fi
'''
}

View file

@ -77,6 +77,7 @@ To use the EC2 cloud module, set up the cloud configuration at
# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
from functools import cmp_to_key
import os
import sys
import stat
@ -1226,7 +1227,7 @@ def get_imageid(vm_):
_t = lambda x: datetime.datetime.strptime(x['creationDate'], '%Y-%m-%dT%H:%M:%S.%fZ')
image_id = sorted(aws.query(params, location=get_location(),
provider=get_provider(), opts=__opts__, sigver='4'),
lambda i, j: salt.utils.compat.cmp(_t(i), _t(j))
key=cmp_to_key(lambda i, j: salt.utils.compat.cmp(_t(i), _t(j)))
)[-1]['imageId']
get_imageid.images[image] = image_id
return image_id

View file

@ -680,23 +680,16 @@ class ProcessManager(object):
class MultiprocessingProcess(multiprocessing.Process, NewStyleClassMixIn):
def __new__(cls, *args, **kwargs):
instance = super(MultiprocessingProcess, cls).__new__(cls)
# Patch the run method at runtime because decorating the run method
# with a function with a similar behavior would be ignored once this
# class'es run method is overridden.
instance._original_run = instance.run
instance.run = instance._run
return instance
def __init__(self, *args, **kwargs):
log_queue = kwargs.pop('log_queue', None)
log_queue_level = kwargs.pop('log_queue_level', None)
super(MultiprocessingProcess, self).__init__(*args, **kwargs)
if salt.utils.platform.is_windows():
# On Windows, subclasses should call super if they define
# __setstate__ and/or __getstate__
self._args_for_getstate = copy.copy(args)
self._kwargs_for_getstate = copy.copy(kwargs)
self.log_queue = kwargs.pop('log_queue', None)
self.log_queue = log_queue
if self.log_queue is None:
self.log_queue = salt.log.setup.get_multiprocessing_logging_queue()
else:
@ -704,16 +697,12 @@ class MultiprocessingProcess(multiprocessing.Process, NewStyleClassMixIn):
# salt.log.setup.get_multiprocessing_logging_queue().
salt.log.setup.set_multiprocessing_logging_queue(self.log_queue)
self.log_queue_level = kwargs.pop('log_queue_level', None)
self.log_queue_level = log_queue_level
if self.log_queue_level is None:
self.log_queue_level = salt.log.setup.get_multiprocessing_logging_level()
else:
salt.log.setup.set_multiprocessing_logging_level(self.log_queue_level)
# Call __init__ from 'multiprocessing.Process' only after removing
# 'log_queue' and 'log_queue_level' from kwargs.
super(MultiprocessingProcess, self).__init__(*args, **kwargs)
self._after_fork_methods = [
(MultiprocessingProcess._setup_process_logging, [self], {}),
]
@ -737,10 +726,6 @@ class MultiprocessingProcess(multiprocessing.Process, NewStyleClassMixIn):
kwargs['log_queue'] = self.log_queue
if 'log_queue_level' not in kwargs:
kwargs['log_queue_level'] = self.log_queue_level
# Remove the version of these in the parent process since
# they are no longer needed.
del self._args_for_getstate
del self._kwargs_for_getstate
return {'args': args,
'kwargs': kwargs,
'_after_fork_methods': self._after_fork_methods,
@ -750,11 +735,11 @@ class MultiprocessingProcess(multiprocessing.Process, NewStyleClassMixIn):
def _setup_process_logging(self):
salt.log.setup.setup_multiprocessing_logging(self.log_queue)
def _run(self):
def run(self):
for method, args, kwargs in self._after_fork_methods:
method(*args, **kwargs)
try:
return self._original_run()
return super(MultiprocessingProcess, self).run()
except SystemExit:
# These are handled by multiprocessing.Process._bootstrap()
raise

View file

@ -172,6 +172,7 @@ def runas(cmdLine, username, password=None, cwd=None):
# Create the environment for the user
env = win32profile.CreateEnvironmentBlock(user_token, False)
hProcess = None
try:
# Start the process in a suspended state.
process_info = salt.platform.win.CreateProcessWithTokenW(
@ -216,7 +217,8 @@ def runas(cmdLine, username, password=None, cwd=None):
stderr = f_err.read()
ret['stderr'] = stderr
finally:
salt.platform.win.kernel32.CloseHandle(hProcess)
if hProcess is not None:
salt.platform.win.kernel32.CloseHandle(hProcess)
win32api.CloseHandle(th)
win32api.CloseHandle(user_token)
if impersonation_token:

View file

@ -179,6 +179,12 @@ class SocketServerRequestHandler(socketserver.StreamRequestHandler):
log.exception(exc)
class TestDaemonStartFailed(Exception):
'''
Simple exception to signal that a test daemon failed to start
'''
class TestDaemon(object):
'''
Set up the master and minion daemons, and run related cases
@ -313,6 +319,7 @@ class TestDaemon(object):
' * {LIGHT_RED}Starting salt-master ... FAILED!\n{ENDC}'.format(**self.colors)
)
sys.stdout.flush()
raise TestDaemonStartFailed()
try:
sys.stdout.write(
@ -349,6 +356,7 @@ class TestDaemon(object):
' * {LIGHT_RED}Starting salt-minion ... FAILED!\n{ENDC}'.format(**self.colors)
)
sys.stdout.flush()
raise TestDaemonStartFailed()
try:
sys.stdout.write(
@ -385,6 +393,7 @@ class TestDaemon(object):
' * {LIGHT_RED}Starting sub salt-minion ... FAILED!\n{ENDC}'.format(**self.colors)
)
sys.stdout.flush()
raise TestDaemonStartFailed()
try:
sys.stdout.write(
@ -422,6 +431,7 @@ class TestDaemon(object):
' * {LIGHT_RED}Starting syndic salt-master ... FAILED!\n{ENDC}'.format(**self.colors)
)
sys.stdout.flush()
raise TestDaemonStartFailed()
try:
sys.stdout.write(
@ -458,6 +468,7 @@ class TestDaemon(object):
' * {LIGHT_RED}Starting salt-syndic ... FAILED!\n{ENDC}'.format(**self.colors)
)
sys.stdout.flush()
raise TestDaemonStartFailed()
if self.parser.options.proxy:
self.minion_targets.add(self.proxy_opts['id'])
@ -496,6 +507,7 @@ class TestDaemon(object):
' * {LIGHT_RED}Starting salt-proxy ... FAILED!\n{ENDC}'.format(**self.colors)
)
sys.stdout.flush()
raise TestDaemonStartFailed()
def start_raet_daemons(self):
'''
@ -1086,23 +1098,32 @@ class TestDaemon(object):
'''
Kill the minion and master processes
'''
if hasattr(self.sub_minion_process, 'terminate'):
self.sub_minion_process.terminate()
else:
log.error('self.sub_minion_process can\'t be terminate.')
try:
if hasattr(self.sub_minion_process, 'terminate'):
self.sub_minion_process.terminate()
else:
log.error('self.sub_minion_process can\'t be terminate.')
except AttributeError:
pass
if hasattr(self.minion_process, 'terminate'):
self.minion_process.terminate()
else:
log.error('self.minion_process can\'t be terminate.')
try:
if hasattr(self.minion_process, 'terminate'):
self.minion_process.terminate()
else:
log.error('self.minion_process can\'t be terminate.')
except AttributeError:
pass
if hasattr(self, 'proxy_process'):
self.proxy_process.terminate()
if hasattr(self.master_process, 'terminate'):
self.master_process.terminate()
else:
log.error('self.master_process can\'t be terminate.')
try:
if hasattr(self.master_process, 'terminate'):
self.master_process.terminate()
else:
log.error('self.master_process can\'t be terminate.')
except AttributeError:
pass
try:
self.syndic_process.terminate()
@ -1112,30 +1133,15 @@ class TestDaemon(object):
self.smaster_process.terminate()
except AttributeError:
pass
#salt.utils.process.clean_proc(self.sub_minion_process, wait_for_kill=50)
#self.sub_minion_process.join()
#salt.utils.process.clean_proc(self.minion_process, wait_for_kill=50)
#self.minion_process.join()
#salt.utils.process.clean_proc(self.master_process, wait_for_kill=50)
#self.master_process.join()
#try:
# salt.utils.process.clean_proc(self.syndic_process, wait_for_kill=50)
# self.syndic_process.join()
#except AttributeError:
# pass
#try:
# salt.utils.process.clean_proc(self.smaster_process, wait_for_kill=50)
# self.smaster_process.join()
#except AttributeError:
# pass
self.log_server.server_close()
self.log_server.shutdown()
self._exit_mockbin()
self._exit_ssh()
self.log_server_process.join()
# Shutdown the multiprocessing logging queue listener
salt_log_setup.shutdown_multiprocessing_logging()
salt_log_setup.shutdown_multiprocessing_logging_listener(daemonizing=True)
# Shutdown the log server
self.log_server.server_close()
self.log_server.shutdown()
self.log_server_process.join()
def pre_setup_minions(self):
'''

View file

@ -108,7 +108,6 @@ class EC2Test(CloudTest):
# check if instance returned with salt installed
self.assertInstanceExists(ret_val)
# Let the instance exist for a bit before destroying it, otherwise the test will fail
self.assertDestroyInstance()

View file

@ -58,7 +58,7 @@ class AzureTest(CloudTest):
Test creating an instance on Azure
'''
# check if instance with salt installed returned
ret_val = self.run_cloud('-p {0} {1}'.format(self.profile_str, self.instance_name), timeout=TIMEOUT)
ret_val = self.run_cloud('-p azure-test {0}'.format(self.instance_name), timeout=TIMEOUT)
self.assertInstanceExists(ret_val)
self.assertDestroyInstance()

View file

@ -5,18 +5,21 @@ Tests for the Openstack Cloud Provider
# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
import logging
from os import path
from time import sleep
import logging
import os
import shutil
# Import Salt Testing libs
from tests.support.case import ShellCase
from tests.support.helpers import generate_random_name, expensiveTest
from tests.support.paths import FILES
from tests.support.runtests import RUNTIME_VARS
# Import Salt Libs
from salt.config import cloud_config, cloud_providers_config
from salt.ext.six.moves import range
from salt.utils.yaml import safe_load
TIMEOUT = 500
@ -26,9 +29,19 @@ log = logging.getLogger(__name__)
class CloudTest(ShellCase):
PROVIDER = ''
REQUIRED_PROVIDER_CONFIG_ITEMS = tuple()
__RE_RUN_DELAY = 15
TMP_PROVIDER_DIR = os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'cloud.providers.d')
__RE_RUN_DELAY = 30
__RE_TRIES = 12
@staticmethod
def clean_cloud_dir(tmp_dir):
'''
Clean the cloud.providers.d tmp directory
'''
# make sure old provider configs are deleted
for i in os.listdir(tmp_dir):
os.remove(os.path.join(tmp_dir, i))
def query_instances(self):
'''
Standardize the data returned from a salt-cloud --query
@ -60,73 +73,64 @@ class CloudTest(ShellCase):
if not instance_name:
instance_name = self.instance_name
# Verify that the instance exists via query
query = self.query_instances()
for tries in range(self.__RE_TRIES):
if self._instance_exists(instance_name, query):
log.debug('Instance "{}" reported after {} seconds'.format(instance_name, tries * self.__RE_RUN_DELAY))
break
else:
sleep(self.__RE_RUN_DELAY)
query = self.query_instances()
# Assert that the last query was successful
self.assertTrue(self._instance_exists(instance_name, query),
'Instance "{}" was not created successfully: |\n\t{}\n\t|`'.format(
instance_name, '\n\t'.join(creation_ret if creation_ret else query)))
log.debug('Instance exists and was created: "{}"'.format(instance_name))
# If it exists but doesn't show up in the creation_ret, there was probably an error during creation
if creation_ret:
self.assertIn(instance_name, [i.strip(': ') for i in creation_ret],
'An error occured during instance creation: |\n\t{}\n\t|'.format(
'\n\t'.join(creation_ret if creation_ret else query)
'\n\t'.join(creation_ret)
))
def _destroy_instance(self):
log.debug('Deleting instance "{}"'.format(self.instance_name))
delete = self.run_cloud('-d {0} --assume-yes'.format(self.instance_name), timeout=TIMEOUT)
# example response: ['gce-config:', '----------', ' gce:', '----------', 'cloud-test-dq4e6c:', 'True', '']
delete_str = ''.join(delete)
log.debug('Deletion status: {}'.format(delete_str))
if any([x in delete_str for x in (
'True',
'was successfully deleted'
)]):
log.debug('Instance "{}" was successfully deleted'.format(self.instance_name))
elif any([x in delete_str for x in (
'shutting-down',
'.delete',
)]):
log.debug('Instance "{}" is cleaning up'.format(self.instance_name))
sleep(30)
else:
log.warning('Instance "{}" may not have been deleted properly'.format(self.instance_name))
# Verify that the instance exists via query
query = self.query_instances()
for tries in range(self.__RE_TRIES):
if self._instance_exists(instance_name, query):
log.debug(
'Instance "{}" reported after {} seconds'.format(instance_name, tries * self.__RE_RUN_DELAY))
break
else:
sleep(self.__RE_RUN_DELAY)
query = self.query_instances()
return delete_str
# Assert that the last query was successful
self.assertTrue(self._instance_exists(instance_name, query),
'Instance "{}" was not created successfully: {}'.format(self.instance_name, ', '.join(query)))
log.debug('Instance exists and was created: "{}"'.format(instance_name))
def assertDestroyInstance(self):
delete_str = self._destroy_instance()
log.debug('Deleting instance "{}"'.format(self.instance_name))
delete_str = self.run_cloud('-d {0} --assume-yes --out=yaml'.format(self.instance_name), timeout=TIMEOUT)
if delete_str:
delete = safe_load('\n'.join(delete_str))
# example response: ['gce-config:', '----------', ' gce:', '----------', 'cloud-test-dq4e6c:', 'True', '']
self.assertIn(self.profile_str, delete)
self.assertIn(self.PROVIDER, delete[self.profile_str])
self.assertIn(self.instance_name, delete[self.profile_str][self.PROVIDER])
# It might take a while to register that deletion has happened with `salt-cloud --query`
delete_status = delete[self.profile_str][self.PROVIDER][self.instance_name]
if isinstance(delete_status, str):
self.assertEqual(delete_status, 'True')
return
elif isinstance(delete_status, dict):
current_state = delete_status.get('currentState')
if current_state:
if current_state.get('ACTION'):
self.assertIn('.delete', current_state.get('ACTION'))
return
else:
self.assertEqual(current_state.get('name'), 'shutting-down')
return
# It's not clear from the delete string that deletion was successful, ask salt-cloud after a delay
query = self.query_instances()
for tries in range(self.__RE_TRIES):
# If the instance doesn't exist, then deletion was a success. Move on
if not self._instance_exists(self.instance_name, query):
log.debug('Instance "{}" reported as deleted after {} seconds'.format(self.instance_name,
tries * self.__RE_RUN_DELAY))
break
else:
# Wait a bit and check again
sleep(self.__RE_RUN_DELAY)
# some instances take a while to report their destruction
for tries in range(6):
if self._instance_exists(query=query):
sleep(30)
log.debug('Instance "{}" still found in query after {} tries: {}'
.format(self.instance_name, tries, query))
query = self.query_instances()
# The instance should be reported as destroyed by the final query, otherwise fail
self.assertFalse(self._instance_exists(query), 'Could not destroy "{}". Delete_str: `{}`'
.format(self.instance_name, delete_str))
log.debug('Instance "{}" no longer exists'.format(self.instance_name))
# The last query should have been successful
self.assertNotIn(self.instance_name, self.query_instances())
@property
def instance_name(self):
@ -147,9 +151,8 @@ class CloudTest(ShellCase):
def provider_config(self):
if not hasattr(self, '_provider_config'):
self._provider_config = cloud_providers_config(
path.join(
FILES,
'conf',
os.path.join(
self.config_dir,
'cloud.providers.d',
self.PROVIDER + '.conf'
)
@ -160,9 +163,8 @@ class CloudTest(ShellCase):
def config(self):
if not hasattr(self, '_config'):
self._config = cloud_config(
path.join(
FILES,
'conf',
os.path.join(
self.config_dir,
'cloud.profiles.d',
self.PROVIDER + '.conf'
)
@ -201,27 +203,43 @@ class CloudTest(ShellCase):
.format(', '.join(missing_conf_item)) +
'\nCheck tests/integration/files/conf/cloud.providers.d/{0}.conf'.format(self.PROVIDER))
self.assertFalse(self._instance_exists(),
'The instance "{}" exists before it was created by the test'.format(self.instance_name))
def tearDown(self):
'''
Clean up after tests, If the instance still exists for any reason, delete it.
Instances should be destroyed before the tearDown, _destroy_instance() should be called exactly
Instances should be destroyed before the tearDown, assertDestroyInstance() should be called exactly
one time in a test for each instance created. This is a failSafe and something went wrong
if the tearDown is where an instance is destroyed.
'''
# Make sure that the instance for sure gets deleted, but fail the test if it happens in the tearDown
instance_deleted_before_teardown = True
for _ in range(12):
if self._instance_exists():
sleep(30)
instance_deleted_before_teardown = False
self._destroy_instance()
destroyed = False
if self._instance_exists():
for tries in range(3):
try:
self.assertDestroyInstance()
self.fail('The instance "{}" was deleted during the tearDown, not the test.'.format(
self.instance_name))
except AssertionError as e:
log.error('Failed to delete instance "{}". Tries: {}\n{}'.format(self.instance_name, tries, str(e)))
if not self._instance_exists():
destroyed = True
break
else:
sleep(30)
self.assertFalse(self._instance_exists(), 'Instance exists after multiple attempts to delete: {}'
.format(self.instance_name))
if not destroyed:
# Destroying instances in the tearDown is a contingency, not the way things should work by default.
self.fail('The Instance "{}" was not deleted after multiple attempts'.format(self.instance_name))
# Destroying instances in the tearDown is a contingency, not the way things should work by default.
self.assertTrue(instance_deleted_before_teardown,
'The Instance "{}" was deleted during the tearDown, not the test.'.format(self.instance_name))
@classmethod
def tearDownClass(cls):
cls.clean_cloud_dir(cls.TMP_PROVIDER_DIR)
@classmethod
def setUpClass(cls):
# clean up before setup
cls.clean_cloud_dir(cls.TMP_PROVIDER_DIR)
# add the provider config for only the cloud we are testing
provider_file = cls.PROVIDER + '.conf'
shutil.copyfile(os.path.join(os.path.join(FILES, 'conf', 'cloud.providers.d'), provider_file),
os.path.join(os.path.join(cls.TMP_PROVIDER_DIR, provider_file)))

View file

@ -1,5 +1,5 @@
azure-test:
provider: azure-config
provider: azurearm-config
image: 'b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu-14_04-LTS-amd64-server-20140724-en-us-30GB'
size: Standard_D1
slot: production

View file

@ -1,6 +1,6 @@
ec2-test:
provider: ec2-config
image: ami-3ecc8f46
image: ''
size: c5.large
sh_username: centos
script_args: '-P'
@ -8,7 +8,7 @@ ec2-test:
ec2-win2012r2-test:
provider: ec2-config
size: c5.large
image: ami-004d6bbd25fdba500
image: ''
smb_port: 445
win_installer: ''
win_username: Administrator
@ -22,7 +22,7 @@ ec2-win2012r2-test:
ec2-win2016-test:
provider: ec2-config
size: c5.large
image: ami-013c9f19b48ddfd08
image: ''
smb_port: 445
win_installer: ''
win_username: Administrator

View file

@ -1,6 +0,0 @@
joyent-test:
provider: joyent-config
size: k4-highcpu-kvm-250M
image: ubuntu-16.04
location: us-east-1
script_args: '-P'

View file

@ -1,4 +1,4 @@
azure-config:
azurearm-config:
driver: azurearm
subscription_id: ''
cleanup_disks: True

View file

@ -1,6 +0,0 @@
joyent-config:
driver: joyent
user: ''
password: ''
private_key: ''
keyname: ''

View file

@ -7,12 +7,13 @@ Set up the Salt multimaster test suite
# Import Python libs
from __future__ import absolute_import, print_function
import copy
import logging
import os
import stat
import shutil
import stat
import sys
import time
import threading
import time
# Import salt tests support dirs
from tests.support.paths import (
@ -32,6 +33,7 @@ from tests.integration import (
SALT_LOG_PORT,
SocketServerRequestHandler,
TestDaemon,
TestDaemonStartFailed,
ThreadedSocketServer,
get_unused_localhost_port,
)
@ -43,6 +45,8 @@ from salt.utils.verify import verify_env
# Import salt tests support libs
from tests.support.processes import SaltMaster, SaltMinion
log = logging.getLogger(__name__)
class MultimasterTestDaemon(TestDaemon):
'''
@ -121,18 +125,47 @@ class MultimasterTestDaemon(TestDaemon):
'''
Kill the minion and master processes
'''
self.sub_minion_process.terminate()
self.minion_process.terminate()
self.sub_master_process.terminate()
self.master_process.terminate()
self.log_server.server_close()
self.log_server.shutdown()
try:
if hasattr(self.sub_minion_process, 'terminate'):
self.sub_minion_process.terminate()
else:
log.error('self.sub_minion_process can\'t be terminate.')
except AttributeError:
pass
try:
if hasattr(self.minion_process, 'terminate'):
self.minion_process.terminate()
else:
log.error('self.minion_process can\'t be terminate.')
except AttributeError:
pass
try:
if hasattr(self.sub_master_process, 'terminate'):
self.sub_master_process.terminate()
else:
log.error('self.sub_master_process can\'t be terminate.')
except AttributeError:
pass
try:
if hasattr(self.master_process, 'terminate'):
self.master_process.terminate()
else:
log.error('self.master_process can\'t be terminate.')
except AttributeError:
pass
self._exit_mockbin()
self._exit_ssh()
self.log_server_process.join()
# Shutdown the multiprocessing logging queue listener
salt_log_setup.shutdown_multiprocessing_logging()
salt_log_setup.shutdown_multiprocessing_logging_listener(daemonizing=True)
# Shutdown the log server
self.log_server.server_close()
self.log_server.shutdown()
self.log_server_process.join()
def start_zeromq_daemons(self):
'''
@ -177,6 +210,7 @@ class MultimasterTestDaemon(TestDaemon):
' * {LIGHT_RED}Starting salt-master ... FAILED!\n{ENDC}'.format(**self.colors)
)
sys.stdout.flush()
raise TestDaemonStartFailed()
# Clone the master key to sub-master's pki dir
for keyfile in ('master.pem', 'master.pub'):
@ -220,6 +254,7 @@ class MultimasterTestDaemon(TestDaemon):
' * {LIGHT_RED}Starting second salt-master ... FAILED!\n{ENDC}'.format(**self.colors)
)
sys.stdout.flush()
raise TestDaemonStartFailed()
try:
sys.stdout.write(
@ -256,6 +291,7 @@ class MultimasterTestDaemon(TestDaemon):
' * {LIGHT_RED}Starting salt-minion ... FAILED!\n{ENDC}'.format(**self.colors)
)
sys.stdout.flush()
raise TestDaemonStartFailed()
try:
sys.stdout.write(
@ -292,6 +328,7 @@ class MultimasterTestDaemon(TestDaemon):
' * {LIGHT_RED}Starting sub salt-minion ... FAILED!\n{ENDC}'.format(**self.colors)
)
sys.stdout.flush()
raise TestDaemonStartFailed()
start_tcp_daemons = start_zeromq_daemons

View file

@ -58,7 +58,7 @@ except ImportError as exc:
pprint.pprint(sys.path)
raise exc
from tests.integration import TestDaemon # pylint: disable=W0403
from tests.integration import TestDaemon, TestDaemonStartFailed # pylint: disable=W0403
from tests.multimaster import MultimasterTestDaemon
import salt.utils.platform
@ -628,65 +628,68 @@ class SaltTestsuiteParser(SaltCoverageTestingParser):
except TypeError:
print_header(' * Setting up Salt daemons for interactive use', top=False)
with TestDaemon(self):
print_header(' * Salt daemons started')
master_conf = TestDaemon.config('master')
minion_conf = TestDaemon.config('minion')
proxy_conf = TestDaemon.config('proxy')
sub_minion_conf = TestDaemon.config('sub_minion')
syndic_conf = TestDaemon.config('syndic')
syndic_master_conf = TestDaemon.config('syndic_master')
try:
with TestDaemon(self):
print_header(' * Salt daemons started')
master_conf = TestDaemon.config('master')
minion_conf = TestDaemon.config('minion')
proxy_conf = TestDaemon.config('proxy')
sub_minion_conf = TestDaemon.config('sub_minion')
syndic_conf = TestDaemon.config('syndic')
syndic_master_conf = TestDaemon.config('syndic_master')
print_header(' * Syndic master configuration values (MoM)', top=False)
print('interface: {0}'.format(syndic_master_conf['interface']))
print('publish port: {0}'.format(syndic_master_conf['publish_port']))
print('return port: {0}'.format(syndic_master_conf['ret_port']))
print('\n')
print_header(' * Syndic master configuration values (MoM)', top=False)
print('interface: {0}'.format(syndic_master_conf['interface']))
print('publish port: {0}'.format(syndic_master_conf['publish_port']))
print('return port: {0}'.format(syndic_master_conf['ret_port']))
print('\n')
print_header(' * Syndic configuration values', top=True)
print('interface: {0}'.format(syndic_conf['interface']))
print('syndic master: {0}'.format(syndic_conf['syndic_master']))
print('syndic master port: {0}'.format(syndic_conf['syndic_master_port']))
print('\n')
print_header(' * Syndic configuration values', top=True)
print('interface: {0}'.format(syndic_conf['interface']))
print('syndic master: {0}'.format(syndic_conf['syndic_master']))
print('syndic master port: {0}'.format(syndic_conf['syndic_master_port']))
print('\n')
print_header(' * Master configuration values', top=True)
print('interface: {0}'.format(master_conf['interface']))
print('publish port: {0}'.format(master_conf['publish_port']))
print('return port: {0}'.format(master_conf['ret_port']))
print('\n')
print_header(' * Master configuration values', top=True)
print('interface: {0}'.format(master_conf['interface']))
print('publish port: {0}'.format(master_conf['publish_port']))
print('return port: {0}'.format(master_conf['ret_port']))
print('\n')
print_header(' * Minion configuration values', top=True)
print('interface: {0}'.format(minion_conf['interface']))
print('master: {0}'.format(minion_conf['master']))
print('master port: {0}'.format(minion_conf['master_port']))
if minion_conf['ipc_mode'] == 'tcp':
print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port']))
print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port']))
print('\n')
print_header(' * Minion configuration values', top=True)
print('interface: {0}'.format(minion_conf['interface']))
print('master: {0}'.format(minion_conf['master']))
print('master port: {0}'.format(minion_conf['master_port']))
if minion_conf['ipc_mode'] == 'tcp':
print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port']))
print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port']))
print('\n')
print_header(' * Sub Minion configuration values', top=True)
print('interface: {0}'.format(sub_minion_conf['interface']))
print('master: {0}'.format(sub_minion_conf['master']))
print('master port: {0}'.format(sub_minion_conf['master_port']))
if sub_minion_conf['ipc_mode'] == 'tcp':
print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port']))
print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port']))
print('\n')
print_header(' * Sub Minion configuration values', top=True)
print('interface: {0}'.format(sub_minion_conf['interface']))
print('master: {0}'.format(sub_minion_conf['master']))
print('master port: {0}'.format(sub_minion_conf['master_port']))
if sub_minion_conf['ipc_mode'] == 'tcp':
print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port']))
print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port']))
print('\n')
print_header(' * Proxy Minion configuration values', top=True)
print('interface: {0}'.format(proxy_conf['interface']))
print('master: {0}'.format(proxy_conf['master']))
print('master port: {0}'.format(proxy_conf['master_port']))
if minion_conf['ipc_mode'] == 'tcp':
print('tcp pub port: {0}'.format(proxy_conf['tcp_pub_port']))
print('tcp pull port: {0}'.format(proxy_conf['tcp_pull_port']))
print('\n')
print_header(' * Proxy Minion configuration values', top=True)
print('interface: {0}'.format(proxy_conf['interface']))
print('master: {0}'.format(proxy_conf['master']))
print('master port: {0}'.format(proxy_conf['master_port']))
if minion_conf['ipc_mode'] == 'tcp':
print('tcp pub port: {0}'.format(proxy_conf['tcp_pub_port']))
print('tcp pull port: {0}'.format(proxy_conf['tcp_pull_port']))
print('\n')
print_header(' Your client configuration is at {0}'.format(TestDaemon.config_location()))
print('To access the minion: salt -c {0} minion test.ping'.format(TestDaemon.config_location()))
print_header(' Your client configuration is at {0}'.format(TestDaemon.config_location()))
print('To access the minion: salt -c {0} minion test.ping'.format(TestDaemon.config_location()))
while True:
time.sleep(1)
while True:
time.sleep(1)
except TestDaemonStartFailed:
self.exit(status=2)
def start_multimaster_daemons_only(self):
if not salt.utils.platform.is_windows():
@ -699,49 +702,52 @@ class SaltTestsuiteParser(SaltCoverageTestingParser):
except TypeError:
print_header(' * Setting up Salt daemons for interactive use', top=False)
with MultimasterTestDaemon(self):
print_header(' * Salt daemons started')
master_conf = MultimasterTestDaemon.config('mm_master')
sub_master_conf = MultimasterTestDaemon.config('mm_sub_master')
minion_conf = MultimasterTestDaemon.config('mm_minion')
sub_minion_conf = MultimasterTestDaemon.config('mm_sub_minion')
try:
with MultimasterTestDaemon(self):
print_header(' * Salt daemons started')
master_conf = MultimasterTestDaemon.config('mm_master')
sub_master_conf = MultimasterTestDaemon.config('mm_sub_master')
minion_conf = MultimasterTestDaemon.config('mm_minion')
sub_minion_conf = MultimasterTestDaemon.config('mm_sub_minion')
print_header(' * Master configuration values', top=True)
print('interface: {0}'.format(master_conf['interface']))
print('publish port: {0}'.format(master_conf['publish_port']))
print('return port: {0}'.format(master_conf['ret_port']))
print('\n')
print_header(' * Master configuration values', top=True)
print('interface: {0}'.format(master_conf['interface']))
print('publish port: {0}'.format(master_conf['publish_port']))
print('return port: {0}'.format(master_conf['ret_port']))
print('\n')
print_header(' * Second master configuration values', top=True)
print('interface: {0}'.format(sub_master_conf['interface']))
print('publish port: {0}'.format(sub_master_conf['publish_port']))
print('return port: {0}'.format(sub_master_conf['ret_port']))
print('\n')
print_header(' * Second master configuration values', top=True)
print('interface: {0}'.format(sub_master_conf['interface']))
print('publish port: {0}'.format(sub_master_conf['publish_port']))
print('return port: {0}'.format(sub_master_conf['ret_port']))
print('\n')
print_header(' * Minion configuration values', top=True)
print('interface: {0}'.format(minion_conf['interface']))
print('masters: {0}'.format(', '.join(minion_conf['master'])))
if minion_conf['ipc_mode'] == 'tcp':
print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port']))
print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port']))
print('\n')
print_header(' * Minion configuration values', top=True)
print('interface: {0}'.format(minion_conf['interface']))
print('masters: {0}'.format(', '.join(minion_conf['master'])))
if minion_conf['ipc_mode'] == 'tcp':
print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port']))
print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port']))
print('\n')
print_header(' * Sub Minion configuration values', top=True)
print('interface: {0}'.format(sub_minion_conf['interface']))
print('masters: {0}'.format(', '.join(sub_minion_conf['master'])))
if sub_minion_conf['ipc_mode'] == 'tcp':
print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port']))
print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port']))
print('\n')
print_header(' * Sub Minion configuration values', top=True)
print('interface: {0}'.format(sub_minion_conf['interface']))
print('masters: {0}'.format(', '.join(sub_minion_conf['master'])))
if sub_minion_conf['ipc_mode'] == 'tcp':
print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port']))
print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port']))
print('\n')
print_header(' Your client configurations are at {0}'.format(
', '.join(MultimasterTestDaemon.config_location())))
print('To access minions from different masters use:')
for location in MultimasterTestDaemon.config_location():
print(' salt -c {0} minion test.ping'.format(location))
print_header(' Your client configurations are at {0}'.format(
', '.join(MultimasterTestDaemon.config_location())))
print('To access minions from different masters use:')
for location in MultimasterTestDaemon.config_location():
print(' salt -c {0} minion test.ping'.format(location))
while True:
time.sleep(1)
while True:
time.sleep(1)
except TestDaemonStartFailed:
self.exit(status=2)
def set_filehandle_limits(self, limits='integration'):
'''
@ -843,37 +849,41 @@ class SaltTestsuiteParser(SaltCoverageTestingParser):
if not self._check_enabled_suites(include_cloud_provider=True, include_proxy=True) and not self.options.name:
return status
with TestDaemon(self):
if self.options.name:
for name in self.options.name:
name = name.strip()
if not name:
continue
if os.path.isfile(name):
if not name.endswith('.py'):
try:
with TestDaemon(self):
if self.options.name:
for name in self.options.name:
name = name.strip()
if not name:
continue
if name.startswith(os.path.join('tests', 'unit')):
if os.path.isfile(name):
if not name.endswith('.py'):
continue
if name.startswith((os.path.join('tests', 'unit'),
os.path.join('tests', 'multimaster'))):
continue
results = self.run_suite(os.path.dirname(name),
name,
suffix=os.path.basename(name),
failfast=self.options.failfast,
load_from_name=False)
status.append(results)
continue
results = self.run_suite(os.path.dirname(name),
name,
suffix=os.path.basename(name),
failfast=self.options.failfast,
load_from_name=False)
if name.startswith(('tests.unit.', 'unit.',
'tests.multimaster.', 'multimaster.')):
continue
results = self.run_suite(
'', name, suffix='test_*.py', load_from_name=True,
failfast=self.options.failfast,
)
status.append(results)
continue
if name.startswith(('tests.unit.', 'unit.',
'tests.multimaster', 'multimaster')):
continue
results = self.run_suite(
'', name, suffix='test_*.py', load_from_name=True,
failfast=self.options.failfast,
)
status.append(results)
return status
for suite in TEST_SUITES:
if suite != 'unit' and suite != 'multimaster' and getattr(self.options, suite):
status.append(self.run_integration_suite(**TEST_SUITES[suite]))
return status
return status
for suite in TEST_SUITES:
if suite != 'unit' and suite != 'multimaster' and getattr(self.options, suite):
status.append(self.run_integration_suite(**TEST_SUITES[suite]))
return status
except TestDaemonStartFailed:
self.exit(status=2)
def run_multimaster_tests(self):
'''
@ -905,30 +915,33 @@ class SaltTestsuiteParser(SaltCoverageTestingParser):
status = []
with MultimasterTestDaemon(self):
if self.options.name:
for name in self.options.name:
name = name.strip()
if not name:
continue
if os.path.isfile(name):
if not name.endswith('.py'):
try:
with MultimasterTestDaemon(self):
if self.options.name:
for name in self.options.name:
name = name.strip()
if not name:
continue
if not name.startswith(os.path.join('tests', 'multimaster')):
if os.path.isfile(name):
if not name.endswith('.py'):
continue
if not name.startswith(os.path.join('tests', 'multimaster')):
continue
results = self.run_suite(os.path.dirname(name),
name,
suffix=os.path.basename(name),
load_from_name=False)
status.append(results)
continue
results = self.run_suite(os.path.dirname(name),
name,
suffix=os.path.basename(name),
load_from_name=False)
if not name.startswith(('tests.multimaster.', 'multimaster.')):
continue
results = self.run_suite('', name, suffix='test_*.py', load_from_name=True)
status.append(results)
continue
if not name.startswith(('tests.multimaster.', 'multimaster.')):
continue
results = self.run_suite('', name, suffix='test_*.py', load_from_name=True)
status.append(results)
return status
status.append(self.run_integration_suite(**TEST_SUITES['multimaster']))
return status
return status
status.append(self.run_integration_suite(**TEST_SUITES['multimaster']))
return status
except TestDaemonStartFailed:
self.exit(status=2)
def run_unit_tests(self):
'''

View file

@ -1065,14 +1065,16 @@ def requires_system_grains(func):
'''
@functools.wraps(func)
def decorator(cls):
if not hasattr(cls, 'run_function'):
raise RuntimeError(
'{0} does not have the \'run_function\' method which is '
'necessary to collect the system grains'.format(
cls.__class__.__name__
if not hasattr(requires_system_grains, '__grains__'):
if not hasattr(cls, 'run_function'):
raise RuntimeError(
'{0} does not have the \'run_function\' method which is '
'necessary to collect the system grains'.format(
cls.__class__.__name__
)
)
)
return func(cls, grains=cls.run_function('grains.items'))
requires_system_grains.__grains__ = func(cls, grains=cls.run_function('grains.items'))
return requires_system_grains.__grains__
return decorator
@ -1100,11 +1102,23 @@ def requires_salt_modules(*names):
)
)
not_found_modules = self.run_function('runtests_helpers.modules_available', names)
if not_found_modules:
if len(not_found_modules) == 1:
self.skipTest('Salt module {0!r} is not available'.format(not_found_modules[0]))
self.skipTest('Salt modules not available: {0!r}'.format(not_found_modules))
if not hasattr(requires_salt_modules, '__available_modules__'):
requires_salt_modules.__available_modules__ = set()
_names = []
for name in names:
if name not in requires_salt_modules.__available_modules__:
_names.append(name)
if _names:
not_found_modules = self.run_function('runtests_helpers.modules_available', _names)
for name in _names:
if name not in not_found_modules:
requires_salt_modules.__available_modules__.add(name)
if not_found_modules:
if len(not_found_modules) == 1:
self.skipTest('Salt module {0!r} is not available'.format(not_found_modules[0]))
self.skipTest('Salt modules not available: {0!r}'.format(not_found_modules))
caller.setUp = setUp
return caller
@ -1120,13 +1134,23 @@ def requires_salt_modules(*names):
)
)
for name in names:
if name not in cls.run_function('sys.doc', [name]):
cls.skipTest(
'Salt module {0!r} is not available'.format(name)
)
break
if not hasattr(requires_salt_modules, '__available_modules__'):
requires_salt_modules.__available_modules__ = set()
_names = []
for name in names:
if name not in requires_salt_modules.__available_modules__:
_names.append(name)
if _names:
not_found_modules = cls.run_function('runtests_helpers.modules_available', _names)
for name in _names:
if name not in not_found_modules:
requires_salt_modules.__available_modules__.add(name)
if not_found_modules:
if len(not_found_modules) == 1:
cls.skipTest('Salt module {0!r} is not available'.format(not_found_modules[0]))
cls.skipTest('Salt modules not available: {0!r}'.format(not_found_modules))
return caller(cls)
return wrapper
return decorator

View file

@ -57,28 +57,51 @@ log = logging.getLogger(__name__)
WEIRD_SIGNAL_NUM = -45654
# Let's setup a global exception hook handler which will log all exceptions
# Store a reference to the original handler
__GLOBAL_EXCEPTION_HANDLER = sys.excepthook
def __global_logging_exception_handler(exc_type, exc_value, exc_traceback):
def __global_logging_exception_handler(exc_type, exc_value, exc_traceback,
_logger=logging.getLogger(__name__),
_stderr=sys.__stderr__,
_format_exception=traceback.format_exception):
'''
This function will log all python exceptions.
'''
if exc_type.__name__ == "KeyboardInterrupt":
# Call the original sys.excepthook
sys.__excepthook__(exc_type, exc_value, exc_traceback)
return
# Log the exception
logging.getLogger(__name__).error(
'An un-handled exception was caught by salt-testing\'s global '
'exception handler:\n{0}: {1}\n{2}'.format(
exc_type.__name__,
exc_value,
''.join(traceback.format_exception(
exc_type, exc_value, exc_traceback
)).strip()
try:
msg = (
'An un-handled exception was caught by salt-testing\'s global exception handler:\n{}: {}\n{}'.format(
exc_type.__name__,
exc_value,
''.join(_format_exception(exc_type, exc_value, exc_traceback)).strip()
)
)
)
except Exception: # pylint: disable=broad-except
msg = (
'An un-handled exception was caught by salt-testing\'s global exception handler:\n{}: {}\n'
'(UNABLE TO FORMAT TRACEBACK)'.format(
exc_type.__name__,
exc_value,
)
)
try:
_logger(__name__).error(msg)
except Exception: # pylint: disable=broad-except
# Python is shutting down and logging has been set to None already
try:
_stderr.write(msg + '\n')
except Exception: # pylint: disable=broad-except
# We have also lost reference to sys.__stderr__ ?!
print(msg)
# Call the original sys.excepthook
__GLOBAL_EXCEPTION_HANDLER(exc_type, exc_value, exc_traceback)
try:
sys.__excepthook__(exc_type, exc_value, exc_traceback)
except Exception: # pylint: disable=broad-except
# Python is shutting down and sys has been set to None already
pass
# Set our own exception handler as the one to use
@ -849,12 +872,9 @@ class SaltTestingParser(optparse.OptionParser):
if children:
log.info('Second run at terminating test suite child processes: %s', children)
helpers.terminate_process(children=children, kill_children=True)
log.info(
'Test suite execution finalized with exit code: {0}'.format(
exit_code
)
)
self.exit(exit_code)
exit_msg = 'Test suite execution finalized with exit code: {}'.format(exit_code)
log.info(exit_msg)
self.exit(status=exit_code, msg=exit_msg + '\n')
def run_suite_in_docker(self):
'''

View file

@ -87,3 +87,28 @@ class EC2TestCase(TestCase, LoaderModuleMockMixin):
)
assert ret['passwordData'] == PASS_DATA
assert ret['password'] == 'testp4ss!'
@patch('salt.cloud.clouds.ec2.config.get_cloud_config_value')
@patch('salt.cloud.clouds.ec2.get_location')
@patch('salt.cloud.clouds.ec2.get_provider')
@patch('salt.cloud.clouds.ec2.aws.query')
def test_get_imageid(self, aws_query, get_provider, get_location, config):
'''
test querying imageid function
'''
vm = {}
ami = 'ami-1234'
config.return_value = 'test/*'
get_location.return_value = 'us-west2'
get_provider.return_value = 'ec2'
aws_query.return_value = [{'imageId': ami}]
# test image filter
self.assertEqual(ec2.get_imageid(vm), ami)
# test ami-image
config.return_value = ami
self.assertEqual(ec2.get_imageid(vm), ami)
# we should have only ran aws.query once when testing the aws filter
aws_query.assert_called_once()

View file

@ -116,6 +116,7 @@ class TestSaltCacheLoader(TestCase):
self.template_dir
)
self.opts = {
'file_buffer_size': 1048576,
'cachedir': self.tempdir,
'file_roots': {
'test': [self.template_dir]
@ -267,6 +268,7 @@ class TestGetTemplate(TestCase):
self.template_dir
)
self.local_opts = {
'file_buffer_size': 1048576,
'cachedir': self.tempdir,
'file_client': 'local',
'file_ignore_regex': None,
@ -600,6 +602,7 @@ class TestJinjaDefaultOptions(TestCase):
TestCase.__init__(self, *args, **kws)
self.local_opts = {
'cachedir': CACHEDIR,
'file_buffer_size': 1048576,
'file_client': 'local',
'file_ignore_regex': None,
'file_ignore_glob': None,
@ -661,6 +664,7 @@ class TestCustomExtensions(TestCase):
super(TestCustomExtensions, self).__init__(*args, **kws)
self.local_opts = {
'cachedir': CACHEDIR,
'file_buffer_size': 1048576,
'file_client': 'local',
'file_ignore_regex': None,
'file_ignore_glob': None,

View file

@ -327,7 +327,7 @@ class TestSignalHandlingMultiprocessingProcess(TestCase):
log_to_mock = 'salt.utils.process.MultiprocessingProcess._setup_process_logging'
with patch(sig_to_mock) as ma, patch(log_to_mock) as mb:
self.sh_proc = salt.utils.process.SignalHandlingMultiprocessingProcess(target=self.no_op_target)
self.sh_proc._run()
self.sh_proc.run()
ma.assert_called()
mb.assert_called()
@ -342,7 +342,7 @@ class TestSignalHandlingMultiprocessingProcess(TestCase):
with patch(sig_to_mock):
with patch(teardown_to_mock) as ma, patch(log_to_mock) as mb:
self.sh_proc = salt.utils.process.SignalHandlingMultiprocessingProcess(target=self.no_op_target)
self.sh_proc._run()
self.sh_proc.run()
ma.assert_called()
mb.assert_called()