mirror of
https://github.com/saltstack/salt.git
synced 2025-04-17 10:10:20 +00:00
Merge pull request #54263 from s0undt3ch/hotfix/test-daemons-2019.2.1
[2019.2.1] Exit test suite if test daemons fail to start
This commit is contained in:
Daniel Wozniak
GitHub
commit
94f77a0da7
3 changed files with 164 additions and 132 deletions
|
|
@ -179,6 +179,12 @@ class SocketServerRequestHandler(socketserver.StreamRequestHandler):
|
|||
log.exception(exc)
|
||||
|
||||
|
||||
class TestDaemonStartFailed(Exception):
|
||||
'''
|
||||
Simple exception to signal that a test daemon failed to start
|
||||
'''
|
||||
|
||||
|
||||
class TestDaemon(object):
|
||||
'''
|
||||
Set up the master and minion daemons, and run related cases
|
||||
|
|
@ -313,6 +319,7 @@ class TestDaemon(object):
|
|||
' * {LIGHT_RED}Starting salt-master ... FAILED!\n{ENDC}'.format(**self.colors)
|
||||
)
|
||||
sys.stdout.flush()
|
||||
raise TestDaemonStartFailed()
|
||||
|
||||
try:
|
||||
sys.stdout.write(
|
||||
|
|
@ -349,6 +356,7 @@ class TestDaemon(object):
|
|||
' * {LIGHT_RED}Starting salt-minion ... FAILED!\n{ENDC}'.format(**self.colors)
|
||||
)
|
||||
sys.stdout.flush()
|
||||
raise TestDaemonStartFailed()
|
||||
|
||||
try:
|
||||
sys.stdout.write(
|
||||
|
|
@ -385,6 +393,7 @@ class TestDaemon(object):
|
|||
' * {LIGHT_RED}Starting sub salt-minion ... FAILED!\n{ENDC}'.format(**self.colors)
|
||||
)
|
||||
sys.stdout.flush()
|
||||
raise TestDaemonStartFailed()
|
||||
|
||||
try:
|
||||
sys.stdout.write(
|
||||
|
|
@ -422,6 +431,7 @@ class TestDaemon(object):
|
|||
' * {LIGHT_RED}Starting syndic salt-master ... FAILED!\n{ENDC}'.format(**self.colors)
|
||||
)
|
||||
sys.stdout.flush()
|
||||
raise TestDaemonStartFailed()
|
||||
|
||||
try:
|
||||
sys.stdout.write(
|
||||
|
|
@ -458,6 +468,7 @@ class TestDaemon(object):
|
|||
' * {LIGHT_RED}Starting salt-syndic ... FAILED!\n{ENDC}'.format(**self.colors)
|
||||
)
|
||||
sys.stdout.flush()
|
||||
raise TestDaemonStartFailed()
|
||||
|
||||
if self.parser.options.proxy:
|
||||
self.minion_targets.add(self.proxy_opts['id'])
|
||||
|
|
@ -496,6 +507,7 @@ class TestDaemon(object):
|
|||
' * {LIGHT_RED}Starting salt-proxy ... FAILED!\n{ENDC}'.format(**self.colors)
|
||||
)
|
||||
sys.stdout.flush()
|
||||
raise TestDaemonStartFailed()
|
||||
|
||||
def start_raet_daemons(self):
|
||||
'''
|
||||
|
|
@ -1086,23 +1098,32 @@ class TestDaemon(object):
|
|||
'''
|
||||
Kill the minion and master processes
|
||||
'''
|
||||
if hasattr(self.sub_minion_process, 'terminate'):
|
||||
self.sub_minion_process.terminate()
|
||||
else:
|
||||
log.error('self.sub_minion_process can\'t be terminate.')
|
||||
try:
|
||||
if hasattr(self.sub_minion_process, 'terminate'):
|
||||
self.sub_minion_process.terminate()
|
||||
else:
|
||||
log.error('self.sub_minion_process can\'t be terminate.')
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
if hasattr(self.minion_process, 'terminate'):
|
||||
self.minion_process.terminate()
|
||||
else:
|
||||
log.error('self.minion_process can\'t be terminate.')
|
||||
try:
|
||||
if hasattr(self.minion_process, 'terminate'):
|
||||
self.minion_process.terminate()
|
||||
else:
|
||||
log.error('self.minion_process can\'t be terminate.')
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
if hasattr(self, 'proxy_process'):
|
||||
self.proxy_process.terminate()
|
||||
|
||||
if hasattr(self.master_process, 'terminate'):
|
||||
self.master_process.terminate()
|
||||
else:
|
||||
log.error('self.master_process can\'t be terminate.')
|
||||
try:
|
||||
if hasattr(self.master_process, 'terminate'):
|
||||
self.master_process.terminate()
|
||||
else:
|
||||
log.error('self.master_process can\'t be terminate.')
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
try:
|
||||
self.syndic_process.terminate()
|
||||
|
|
@ -1112,30 +1133,15 @@ class TestDaemon(object):
|
|||
self.smaster_process.terminate()
|
||||
except AttributeError:
|
||||
pass
|
||||
#salt.utils.process.clean_proc(self.sub_minion_process, wait_for_kill=50)
|
||||
#self.sub_minion_process.join()
|
||||
#salt.utils.process.clean_proc(self.minion_process, wait_for_kill=50)
|
||||
#self.minion_process.join()
|
||||
#salt.utils.process.clean_proc(self.master_process, wait_for_kill=50)
|
||||
#self.master_process.join()
|
||||
#try:
|
||||
# salt.utils.process.clean_proc(self.syndic_process, wait_for_kill=50)
|
||||
# self.syndic_process.join()
|
||||
#except AttributeError:
|
||||
# pass
|
||||
#try:
|
||||
# salt.utils.process.clean_proc(self.smaster_process, wait_for_kill=50)
|
||||
# self.smaster_process.join()
|
||||
#except AttributeError:
|
||||
# pass
|
||||
self.log_server.server_close()
|
||||
self.log_server.shutdown()
|
||||
self._exit_mockbin()
|
||||
self._exit_ssh()
|
||||
self.log_server_process.join()
|
||||
# Shutdown the multiprocessing logging queue listener
|
||||
salt_log_setup.shutdown_multiprocessing_logging()
|
||||
salt_log_setup.shutdown_multiprocessing_logging_listener(daemonizing=True)
|
||||
# Shutdown the log server
|
||||
self.log_server.server_close()
|
||||
self.log_server.shutdown()
|
||||
self.log_server_process.join()
|
||||
|
||||
def pre_setup_minions(self):
|
||||
'''
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ except ImportError as exc:
|
|||
pprint.pprint(sys.path)
|
||||
raise exc
|
||||
|
||||
from tests.integration import TestDaemon # pylint: disable=W0403
|
||||
from tests.integration import TestDaemon, TestDaemonStartFailed # pylint: disable=W0403
|
||||
import salt.utils.platform
|
||||
|
||||
if not salt.utils.platform.is_windows():
|
||||
|
|
@ -610,65 +610,68 @@ class SaltTestsuiteParser(SaltCoverageTestingParser):
|
|||
except TypeError:
|
||||
print_header(' * Setting up Salt daemons for interactive use', top=False)
|
||||
|
||||
with TestDaemon(self):
|
||||
print_header(' * Salt daemons started')
|
||||
master_conf = TestDaemon.config('master')
|
||||
minion_conf = TestDaemon.config('minion')
|
||||
proxy_conf = TestDaemon.config('proxy')
|
||||
sub_minion_conf = TestDaemon.config('sub_minion')
|
||||
syndic_conf = TestDaemon.config('syndic')
|
||||
syndic_master_conf = TestDaemon.config('syndic_master')
|
||||
try:
|
||||
with TestDaemon(self):
|
||||
print_header(' * Salt daemons started')
|
||||
master_conf = TestDaemon.config('master')
|
||||
minion_conf = TestDaemon.config('minion')
|
||||
proxy_conf = TestDaemon.config('proxy')
|
||||
sub_minion_conf = TestDaemon.config('sub_minion')
|
||||
syndic_conf = TestDaemon.config('syndic')
|
||||
syndic_master_conf = TestDaemon.config('syndic_master')
|
||||
|
||||
print_header(' * Syndic master configuration values (MoM)', top=False)
|
||||
print('interface: {0}'.format(syndic_master_conf['interface']))
|
||||
print('publish port: {0}'.format(syndic_master_conf['publish_port']))
|
||||
print('return port: {0}'.format(syndic_master_conf['ret_port']))
|
||||
print('\n')
|
||||
print_header(' * Syndic master configuration values (MoM)', top=False)
|
||||
print('interface: {0}'.format(syndic_master_conf['interface']))
|
||||
print('publish port: {0}'.format(syndic_master_conf['publish_port']))
|
||||
print('return port: {0}'.format(syndic_master_conf['ret_port']))
|
||||
print('\n')
|
||||
|
||||
print_header(' * Syndic configuration values', top=True)
|
||||
print('interface: {0}'.format(syndic_conf['interface']))
|
||||
print('syndic master: {0}'.format(syndic_conf['syndic_master']))
|
||||
print('syndic master port: {0}'.format(syndic_conf['syndic_master_port']))
|
||||
print('\n')
|
||||
print_header(' * Syndic configuration values', top=True)
|
||||
print('interface: {0}'.format(syndic_conf['interface']))
|
||||
print('syndic master: {0}'.format(syndic_conf['syndic_master']))
|
||||
print('syndic master port: {0}'.format(syndic_conf['syndic_master_port']))
|
||||
print('\n')
|
||||
|
||||
print_header(' * Master configuration values', top=True)
|
||||
print('interface: {0}'.format(master_conf['interface']))
|
||||
print('publish port: {0}'.format(master_conf['publish_port']))
|
||||
print('return port: {0}'.format(master_conf['ret_port']))
|
||||
print('\n')
|
||||
print_header(' * Master configuration values', top=True)
|
||||
print('interface: {0}'.format(master_conf['interface']))
|
||||
print('publish port: {0}'.format(master_conf['publish_port']))
|
||||
print('return port: {0}'.format(master_conf['ret_port']))
|
||||
print('\n')
|
||||
|
||||
print_header(' * Minion configuration values', top=True)
|
||||
print('interface: {0}'.format(minion_conf['interface']))
|
||||
print('master: {0}'.format(minion_conf['master']))
|
||||
print('master port: {0}'.format(minion_conf['master_port']))
|
||||
if minion_conf['ipc_mode'] == 'tcp':
|
||||
print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port']))
|
||||
print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port']))
|
||||
print('\n')
|
||||
print_header(' * Minion configuration values', top=True)
|
||||
print('interface: {0}'.format(minion_conf['interface']))
|
||||
print('master: {0}'.format(minion_conf['master']))
|
||||
print('master port: {0}'.format(minion_conf['master_port']))
|
||||
if minion_conf['ipc_mode'] == 'tcp':
|
||||
print('tcp pub port: {0}'.format(minion_conf['tcp_pub_port']))
|
||||
print('tcp pull port: {0}'.format(minion_conf['tcp_pull_port']))
|
||||
print('\n')
|
||||
|
||||
print_header(' * Sub Minion configuration values', top=True)
|
||||
print('interface: {0}'.format(sub_minion_conf['interface']))
|
||||
print('master: {0}'.format(sub_minion_conf['master']))
|
||||
print('master port: {0}'.format(sub_minion_conf['master_port']))
|
||||
if sub_minion_conf['ipc_mode'] == 'tcp':
|
||||
print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port']))
|
||||
print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port']))
|
||||
print('\n')
|
||||
print_header(' * Sub Minion configuration values', top=True)
|
||||
print('interface: {0}'.format(sub_minion_conf['interface']))
|
||||
print('master: {0}'.format(sub_minion_conf['master']))
|
||||
print('master port: {0}'.format(sub_minion_conf['master_port']))
|
||||
if sub_minion_conf['ipc_mode'] == 'tcp':
|
||||
print('tcp pub port: {0}'.format(sub_minion_conf['tcp_pub_port']))
|
||||
print('tcp pull port: {0}'.format(sub_minion_conf['tcp_pull_port']))
|
||||
print('\n')
|
||||
|
||||
print_header(' * Proxy Minion configuration values', top=True)
|
||||
print('interface: {0}'.format(proxy_conf['interface']))
|
||||
print('master: {0}'.format(proxy_conf['master']))
|
||||
print('master port: {0}'.format(proxy_conf['master_port']))
|
||||
if minion_conf['ipc_mode'] == 'tcp':
|
||||
print('tcp pub port: {0}'.format(proxy_conf['tcp_pub_port']))
|
||||
print('tcp pull port: {0}'.format(proxy_conf['tcp_pull_port']))
|
||||
print('\n')
|
||||
print_header(' * Proxy Minion configuration values', top=True)
|
||||
print('interface: {0}'.format(proxy_conf['interface']))
|
||||
print('master: {0}'.format(proxy_conf['master']))
|
||||
print('master port: {0}'.format(proxy_conf['master_port']))
|
||||
if minion_conf['ipc_mode'] == 'tcp':
|
||||
print('tcp pub port: {0}'.format(proxy_conf['tcp_pub_port']))
|
||||
print('tcp pull port: {0}'.format(proxy_conf['tcp_pull_port']))
|
||||
print('\n')
|
||||
|
||||
print_header(' Your client configuration is at {0}'.format(TestDaemon.config_location()))
|
||||
print('To access the minion: salt -c {0} minion test.ping'.format(TestDaemon.config_location()))
|
||||
print_header(' Your client configuration is at {0}'.format(TestDaemon.config_location()))
|
||||
print('To access the minion: salt -c {0} minion test.ping'.format(TestDaemon.config_location()))
|
||||
|
||||
while True:
|
||||
time.sleep(1)
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except TestDaemonStartFailed:
|
||||
self.exit(status=2)
|
||||
|
||||
def set_filehandle_limits(self, limits='integration'):
|
||||
'''
|
||||
|
|
@ -768,36 +771,39 @@ class SaltTestsuiteParser(SaltCoverageTestingParser):
|
|||
if not self._check_enabled_suites(include_cloud_provider=True, include_proxy=True) and not self.options.name:
|
||||
return status
|
||||
|
||||
with TestDaemon(self):
|
||||
if self.options.name:
|
||||
for name in self.options.name:
|
||||
name = name.strip()
|
||||
if not name:
|
||||
continue
|
||||
if os.path.isfile(name):
|
||||
if not name.endswith('.py'):
|
||||
try:
|
||||
with TestDaemon(self):
|
||||
if self.options.name:
|
||||
for name in self.options.name:
|
||||
name = name.strip()
|
||||
if not name:
|
||||
continue
|
||||
if name.startswith(os.path.join('tests', 'unit')):
|
||||
if os.path.isfile(name):
|
||||
if not name.endswith('.py'):
|
||||
continue
|
||||
if name.startswith(os.path.join('tests', 'unit')):
|
||||
continue
|
||||
results = self.run_suite(os.path.dirname(name),
|
||||
name,
|
||||
suffix=os.path.basename(name),
|
||||
failfast=self.options.failfast,
|
||||
load_from_name=False)
|
||||
status.append(results)
|
||||
continue
|
||||
results = self.run_suite(os.path.dirname(name),
|
||||
name,
|
||||
suffix=os.path.basename(name),
|
||||
failfast=self.options.failfast,
|
||||
load_from_name=False)
|
||||
if name.startswith(('tests.unit.', 'unit.')):
|
||||
continue
|
||||
results = self.run_suite(
|
||||
'', name, suffix='test_*.py', load_from_name=True,
|
||||
failfast=self.options.failfast,
|
||||
)
|
||||
status.append(results)
|
||||
continue
|
||||
if name.startswith(('tests.unit.', 'unit.')):
|
||||
continue
|
||||
results = self.run_suite(
|
||||
'', name, suffix='test_*.py', load_from_name=True,
|
||||
failfast=self.options.failfast,
|
||||
)
|
||||
status.append(results)
|
||||
return status
|
||||
for suite in TEST_SUITES:
|
||||
if suite != 'unit' and getattr(self.options, suite):
|
||||
status.append(self.run_integration_suite(**TEST_SUITES[suite]))
|
||||
return status
|
||||
return status
|
||||
for suite in TEST_SUITES:
|
||||
if suite != 'unit' and getattr(self.options, suite):
|
||||
status.append(self.run_integration_suite(**TEST_SUITES[suite]))
|
||||
return status
|
||||
except TestDaemonStartFailed:
|
||||
self.exit(status=2)
|
||||
|
||||
def run_unit_tests(self):
|
||||
'''
|
||||
|
|
|
|||
|
|
@ -57,28 +57,51 @@ log = logging.getLogger(__name__)
|
|||
WEIRD_SIGNAL_NUM = -45654
|
||||
|
||||
|
||||
# Let's setup a global exception hook handler which will log all exceptions
|
||||
# Store a reference to the original handler
|
||||
__GLOBAL_EXCEPTION_HANDLER = sys.excepthook
|
||||
|
||||
|
||||
def __global_logging_exception_handler(exc_type, exc_value, exc_traceback):
|
||||
def __global_logging_exception_handler(exc_type, exc_value, exc_traceback,
|
||||
_logger=logging.getLogger(__name__),
|
||||
_stderr=sys.__stderr__,
|
||||
_format_exception=traceback.format_exception):
|
||||
'''
|
||||
This function will log all python exceptions.
|
||||
'''
|
||||
if exc_type.__name__ == "KeyboardInterrupt":
|
||||
# Call the original sys.excepthook
|
||||
sys.__excepthook__(exc_type, exc_value, exc_traceback)
|
||||
return
|
||||
|
||||
# Log the exception
|
||||
logging.getLogger(__name__).error(
|
||||
'An un-handled exception was caught by salt-testing\'s global '
|
||||
'exception handler:\n{0}: {1}\n{2}'.format(
|
||||
exc_type.__name__,
|
||||
exc_value,
|
||||
''.join(traceback.format_exception(
|
||||
exc_type, exc_value, exc_traceback
|
||||
)).strip()
|
||||
try:
|
||||
msg = (
|
||||
'An un-handled exception was caught by salt-testing\'s global exception handler:\n{}: {}\n{}'.format(
|
||||
exc_type.__name__,
|
||||
exc_value,
|
||||
''.join(_format_exception(exc_type, exc_value, exc_traceback)).strip()
|
||||
)
|
||||
)
|
||||
)
|
||||
except Exception: # pylint: disable=broad-except
|
||||
msg = (
|
||||
'An un-handled exception was caught by salt-testing\'s global exception handler:\n{}: {}\n'
|
||||
'(UNABLE TO FORMAT TRACEBACK)'.format(
|
||||
exc_type.__name__,
|
||||
exc_value,
|
||||
)
|
||||
)
|
||||
try:
|
||||
_logger(__name__).error(msg)
|
||||
except Exception: # pylint: disable=broad-except
|
||||
# Python is shutting down and logging has been set to None already
|
||||
try:
|
||||
_stderr.write(msg + '\n')
|
||||
except Exception: # pylint: disable=broad-except
|
||||
# We have also lost reference to sys.__stderr__ ?!
|
||||
print(msg)
|
||||
|
||||
# Call the original sys.excepthook
|
||||
__GLOBAL_EXCEPTION_HANDLER(exc_type, exc_value, exc_traceback)
|
||||
try:
|
||||
sys.__excepthook__(exc_type, exc_value, exc_traceback)
|
||||
except Exception: # pylint: disable=broad-except
|
||||
# Python is shutting down and sys has been set to None already
|
||||
pass
|
||||
|
||||
|
||||
# Set our own exception handler as the one to use
|
||||
|
|
@ -848,12 +871,9 @@ class SaltTestingParser(optparse.OptionParser):
|
|||
if children:
|
||||
log.info('Second run at terminating test suite child processes: %s', children)
|
||||
helpers.terminate_process(children=children, kill_children=True)
|
||||
log.info(
|
||||
'Test suite execution finalized with exit code: {0}'.format(
|
||||
exit_code
|
||||
)
|
||||
)
|
||||
self.exit(exit_code)
|
||||
exit_msg = 'Test suite execution finalized with exit code: {}'.format(exit_code)
|
||||
log.info(exit_msg)
|
||||
self.exit(status=exit_code, msg=exit_msg + '\n')
|
||||
|
||||
def run_suite_in_docker(self):
|
||||
'''
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue