* Improve init script: specifically manage salt configurations rather than arbitrary salt processes (#32666)

* * Improve init script: specifically manage salt configurations rather than arbitrary salt processes

Unfortunately SysV init scripts tend to rummage through PIDs filtering for
appropriate processes to manage.  Unfortunately the filters are usually weak
and don't account for similar processes run by other users, PIDs of dead
processes being re-used for completely different executables, etc..  These
weaknesses can result in killing unrelated processes with potentially serious
results.

These improvements to the SysV init script is a complete rewrite with the
following improvements:

  * Specifically manage individual salt configurations rather than looking for
    salt minion-like processes.
  * Obtain salt minion information from the salt configuration - use the
    information to manage the specifically configured process.
  * Drop all of the platform-specific helper functions that allow the
    previously-mentioned weaknesses.

    + Unfortunately this means that the output information may not match the
      specific platform (this could easily be corrected).

  * Now can manage multiple salt processes started by different users

    + Unfortunately starts/stops/restarts as a group and is unable to manage
      them both as a group or as individual processes (this could easily be
      corrected)

The new initscript also allows various control variables to be overridden by
environment variables or through settings put in ``/etc/sysconf/salt`` or
``/etc/default/salt``.

:SALTMINION_DEBUG: Dump each line expansion before execution, output system
                   information on failure.  Default: unset

:SALTMINION_BINDIR: Location of ``salt-minion``, ``salt-call`` and other
                    executables.  Default: ``/usr/bin``

:SALTMINION_SYSCONFDIR: The parent directory for the ``salt`` configuration
                        directory and the ``sysconfig`` or ``default``
                        directory.

* Add lines that went missing in the rebase+squash
This commit is contained in:
plastikos 2016-05-09 11:57:13 -06:00 committed by Mike Place
parent f354f68b64
commit 0efbbcd17f
5 changed files with 1063 additions and 117 deletions

View file

@ -24,133 +24,301 @@
#
# processname: /usr/bin/salt-minion
# Allow these to be overridden for tests
: ${SALTMINION_BINDIR:=/usr/bin}
: ${SALTMINION_SYSCONFDIR:=/etc}
: ${SALTMINION_PYTHON:=python} # Python must be 2.6 or newer
DEBIAN_VERSION=/etc/debian_version
SUSE_RELEASE=/etc/SuSE-release
# Source function library.
if [ -f $DEBIAN_VERSION ]; then
break
elif [ -f $SUSE_RELEASE -a -r /etc/rc.status ]; then
. /etc/rc.status
else
. /etc/rc.d/init.d/functions
# Default values (can be overridden in settings file)
: ${USER:=$(id -nu)}
SALTMINION="${SALTMINION_BINDIR}/salt-minion"
SALTCALL="${SALTMINION_BINDIR}/salt-call"
# SALTMINION_CONFIGS are newline-separated entries of: MINION_USER CONFIG_DIR
: ${SALTMINION_CONFIGS:="
$USER ${SALTMINION_SYSCONFDIR}/salt
"}
SALTMINION_ARGS=""
SALTMINION_TIMEOUT=30
SALTMINION_TICK=1
SERVICE="salt-minion"
PROCESS="salt-minion"
# Read in settings file
if [ -f "${SALTMINION_SYSCONFDIR}/default/salt" ]; then
. "${SALTMINION_SYSCONFDIR}/default/salt"
elif [ -f "${SALTMINION_SYSCONFDIR}/sysconfig/salt" ]; then
. "${SALTMINION_SYSCONFDIR}/sysconfig/salt"
fi
# Default values (can be overridden below)
SALTMINION=/usr/bin/salt-minion
PYTHON=/usr/bin/python
MINION_ARGS=""
if [ -f /etc/default/salt ]; then
. /etc/default/salt
fi
SERVICE=salt-minion
PROCESS=salt-minion
RETVAL=0
NS_NOTRIM="--notrim"
ERROR_TO_DEVNULL="/dev/null"
start() {
echo -n $"Starting salt-minion daemon: "
if [ -f $SUSE_RELEASE ]; then
startproc -f -p /var/run/$SERVICE.pid $SALTMINION -d $MINION_ARGS
rc_status -v
elif [ -e $DEBIAN_VERSION ]; then
if [ -f $LOCKFILE ]; then
echo -n "already started, lock file found"
RETVAL=1
elif $PYTHON $SALTMINION -d $MINION_ARGS >& /dev/null; then
echo -n "OK"
RETVAL=0
fi
_su_cmd() {
local user="$1"
shift
if [ "X$USER" = "X$user" ]; then
eval $1
else
if [[ ! -z "$(pidofproc -p /var/run/$SERVICE.pid $PROCESS)" ]]; then
RETVAL=$?
echo -n "already running"
else
daemon --check $SERVICE $SALTMINION -d $MINION_ARGS
RETVAL=$?
[ $RETVAL -eq 0 ] && touch /var/lock/subsys/$SERVICE
echo
return $RETVAL
fi
su -l -c "$1" "$user"
fi
RETVAL=$?
echo
return $RETVAL
}
stop() {
echo -n $"Stopping salt-minion daemon: "
if [ -f $SUSE_RELEASE ]; then
killproc -TERM $SALTMINION
rc_status -v
RETVAL=$?
elif [ -f $DEBIAN_VERSION ]; then
# Added this since Debian's start-stop-daemon doesn't support spawned processes
if ps -ef | grep "$PYTHON $SALTMINION" | grep -v grep | awk '{print $2}' | xargs kill &> /dev/null; then
echo -n "OK"
RETVAL=0
else
echo -n "Daemon is not started"
RETVAL=1
fi
else
killproc $PROCESS
RETVAL=$?
[ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$SERVICE
# tidy up any rogue processes:
PROCS=`ps -ef | grep "$SALTMINION" | grep -v grep | awk '{print $2}'`
if [ -n "$PROCS" ]; then
kill $PROCS &> /dev/null
sleep 1
PROCS=`ps -ef | grep "$SALTMINION" | grep -v grep | awk '{print $2}'`
if [ -n "$PROCS" ]; then
kill -9 $PROCS &> /dev/null
_get_pid() {
netstat $NS_NOTRIM -ap --protocol=unix 2>$ERROR_TO_DEVNULL \
| sed -r -e "\|\s${SOCK_DIR}/minion_event_${MINION_ID_HASH}_pub\.ipc$|"'!d; s|/.*||; s/.*\s//;'
}
_is_running() {
[ -n "$(_get_pid)" ]
}
_get_salt_config_value() {
_su_cmd \
"$MINION_USER" \
"\"$SALTMINION_PYTHON\" \
\"$SALTCALL\" \
-c \"$CONFIG_DIR\" \
--no-color \
--local config.get \
\"$1\" \
" \
2>$ERROR_TO_DEVNULL \
| sed -r -e '2!d; s/^\s*//;'
}
_make_id_hash() {
# $1 - minion_id
local hasher=''
case "$(_get_salt_config_value hash_type)" in
(md5) hasher="md5sum";;
(sha1) hasher="sha1sum";;
(sha224) hasher="sha224sum";;
(sha256) hasher="sha256sum";;
(sha384) hasher="sha384sum";;
(sha512) hasher="sha512sum";;
(*) echo "ERROR: No salt hash_type specified";;
esac
if [ -n "$hasher" ]; then
printf "$1" | "$hasher" | cut -c 1-10
fi
}
start() {
# $1 - config dir
local retval=0
if _is_running; then
echo "Service $SERVICE:$MINION_USER:$MINION_ID already running"
return 0
fi
echo -n "Starting $SERVICE:$MINION_USER:$MINION_ID daemon: "
_su_cmd \
"$MINION_USER" \
"\"$SALTMINION_PYTHON\" \
\"$SALTMINION\" \
-c \"$CONFIG_DIR\" \
-d $SALTMINION_ARGS \
${SALTMINION_DEBUG:+-l debug} \
" \
2>$ERROR_TO_DEVNULL \
|| retval=$?
if [ 0 -eq "$retval" ]; then
local endtime=$(($(date '+%s')+$SALTMINION_TIMEOUT))
while ! _is_running; do
if [ "$endtime" -lt "$(date '+%s')" ]; then
echo -n "TIMEOUT "
retval=1
break
fi
sleep $SALTMINION_TICK
done
fi
if [ 0 -eq "$retval" ]; then
echo -n "OK"
else
echo -n "FAIL"
if [ -n "$SALTMINION_DEBUG" ]; then
printf "\nPROCESSES:\n" >&2
ps wwwaxu | grep '[s]alt-minion' >&2
printf "\nSOCKETS:\n" >&2
netstat $NS_NOTRIM -ap --protocol=unix | grep 'salt.*minion' >&2
printf "\nLOG_FILE:\n" >&2
tail -n 20 "$LOG_FILE" >&2
printf "\nENVIRONMENT:\n" >&2
env >&2
fi
fi
echo
return $retval
}
stop() {
# $1 - config dir
local retval=0
if ! _is_running; then
echo "Service $SERVICE:$MINION_USER:$MINION_ID is not running"
return 0
fi
echo -n "Stopping $SERVICE:$MINION_USER:$MINION_ID daemon: "
local pid="$(_get_pid)"
# pid below is intentionally not quoted in case there are *multiple*
# minions running with the same configuration.
_su_cmd "$MINION_USER" "kill -TERM $pid 2>/dev/null" || retval=$?
if [ 0 -eq "$retval" ]; then
local endtime=$(($(date '+%s')+$SALTMINION_TIMEOUT))
while _is_running; do
if [ "$endtime" -lt "$(date '+%s')" ]; then
# Try one more time with a big hammer
_su_cmd "$MINION_USER" "kill -KILL $pid 2>/dev/null" || :
sleep $SALTMINION_TICK
if _is_running; then
echo -n "TIMEOUT "
retval=1
fi
break
fi
sleep 1
done
fi
if [ 0 -eq "$retval" ]; then
rm -f "$PID_FILE"
echo -n "OK"
else
echo -n "FAIL"
fi
echo
return $retval
}
status() {
local retval=0
local pid="$(_get_pid)"
if [ -n "$pid" ]; then
echo "$SERVICE:$MINION_USER:$MINION_ID is running: $pid"
else
retval=3
echo "$SERVICE:$MINION_USER:$MINION_ID is stopped."
if [ -e "$PID_FILE" ]; then
echo "$SERVICE:$MINION_USER:$MINION_ID has orphaned pid file: $PID_FILE."
retval=1
fi
fi
return $retval
}
restart() {
stop
start
# $1 - config dir
stop "$1"
start "$1"
}
# See how we were called.
case "$1" in
start|stop|restart)
$1
main() {
if [ -n "$SALTMINION_DEBUG" ]; then
set -x
ERROR_TO_DEVNULL="&2"
fi
# Check to see if --notrim is a valid netstat option
if netstat --notrim 2>&1 >/dev/null | grep -q 'unrecognized'; then
NS_NOTRIM=''
fi
# Pre-filter for unhandled commands
case "$1" in
(start|stop|status|restart|condrestart|try-restart|reload) ;;
(*)
echo "Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload}"
exit 2
;;
status)
if [ -f $SUSE_RELEASE ]; then
echo -n "Checking for service salt-minion "
checkproc $SALTMINION
rc_status -v
elif [ -f $DEBIAN_VERSION ]; then
if [ -f $LOCKFILE ]; then
RETVAL=0
echo "salt-minion is running."
else
RETVAL=1
echo "salt-minion is stopped."
fi
else
status $PROCESS
RETVAL=$?
esac
while read MINION_USER CONFIG_DIR; do
if [ -z "$CONFIG_DIR" ]; then
continue
fi
;;
condrestart)
[ -f $LOCKFILE ] && restart || :
;;
reload)
echo "can't reload configuration, you have to restart it"
RETVAL=1
;;
*)
echo $"Usage: $0 {start|stop|status|restart|condrestart|reload}"
exit 1
;;
esac
exit $RETVAL
if ! [ -d "$CONFIG_DIR" ]; then
echo "ERROR: non-existent $SERVICE config directory: $CONFIG_DIR"
RETVAL=1
continue
fi
SOCK_DIR="$(_get_salt_config_value sock_dir)"
PID_FILE="$(_get_salt_config_value pidfile)"
LOG_FILE="$(_get_salt_config_value log_file)"
MINION_ID="$(_get_salt_config_value id)"
MINION_ID_HASH="$(_make_id_hash "$MINION_ID")"
if [ \
-z "$SOCK_DIR" \
-o -z "$PID_FILE" \
-o -z "$LOG_FILE" \
-o -z "$MINION_ID" \
-o -z "$MINION_ID_HASH" \
]; then
echo "ERROR: Unable to look-up config values for $CONFIG_DIR"
RETVAL=1
continue
fi
# See how we were called.
case "$1" in
(start|stop|restart|status)
"$1" || RETVAL=$?
;;
(condrestart|try-restart)
if ! _is_running; then
RETVAL=7
else
stop
start || RETVAL=$?
fi
;;
(reload)
echo "Can't reload $SERVICE configuration - you must restart it"
RETVAL=3
;;
(*)
echo "Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload}"
RETVAL=2
;;
esac
done <<EOF
$SALTMINION_CONFIGS
EOF
exit $RETVAL
}
if [ "$#" = 0 ]; then
main
else
main "$@"
fi

View file

@ -508,10 +508,10 @@ class TestDaemon(object):
running_tests_user = pwd.getpwuid(os.getuid()).pw_name
master_opts = salt.config._read_conf_file(os.path.join(CONF_DIR, 'master'))
master_opts['user'] = running_tests_user
tests_know_hosts_file = os.path.join(TMP_CONF_DIR, 'salt_ssh_known_hosts')
with salt.utils.fopen(tests_know_hosts_file, 'w') as known_hosts:
tests_known_hosts_file = os.path.join(TMP_CONF_DIR, 'salt_ssh_known_hosts')
with salt.utils.fopen(tests_known_hosts_file, 'w') as known_hosts:
known_hosts.write('')
master_opts['known_hosts_file'] = tests_know_hosts_file
master_opts['known_hosts_file'] = tests_known_hosts_file
minion_config_path = os.path.join(CONF_DIR, 'minion')
minion_opts = salt.config._read_conf_file(minion_config_path)
@ -1077,7 +1077,7 @@ class AdaptedConfigurationTestCaseMixIn(object):
@property
def master_opts(self):
'''
Return the options used for the minion
Return the options used for the master
'''
return self.get_config('master')

View file

@ -10,9 +10,14 @@
# Import python libs
from __future__ import absolute_import
import os
import sys
import getpass
import platform
import yaml
import signal
import shutil
import tempfile
import logging
# Import Salt Testing libs
from salttesting.helpers import ensure_in_syspath
@ -20,12 +25,36 @@ ensure_in_syspath('../../')
# Import salt libs
import integration
from integration.utils import testprogram
import salt.utils
import salt.defaults.exitcodes
log = logging.getLogger(__name__)
DEBUG = True
class MinionTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn):
'''
Various integration tests for the salt-minion executable.
'''
_call_binary_ = 'salt-minion'
_test_dir = None
_test_minions = (
'minion',
'subminion',
)
def setUp(self):
# Setup for scripts
self._test_dir = tempfile.mkdtemp(prefix='salt-testdaemon-')
def tearDown(self):
# shutdown for scripts
if self._test_dir and os.path.sep == self._test_dir[0]:
shutil.rmtree(self._test_dir)
self._test_dir = None
def test_issue_7754(self):
old_cwd = os.getcwd()
@ -75,7 +104,156 @@ class MinionTest(integration.ShellCase, integration.ShellCaseCommonTestsMixIn):
if os.path.isdir(config_dir):
shutil.rmtree(config_dir)
def _run_initscript(
self,
init_script,
minions,
minion_running,
action,
exitstatus=None,
message=''
):
'''
Wrapper that runs the initscript for the configured minions and
verifies the results.
'''
ret = init_script.run(
[action],
catch_stderr=True,
with_retcode=True,
timeout=90,
)
# Check minion state
for minion in minions:
self.assertEqual(
minion.is_running(),
minion_running,
'Minion "{0}" must be {1} and is not.\nSTDOUT:{2}\nSTDERR:{3}'.format(
minion.name,
["stopped", "running"][minion_running],
'\nSTDOUT:'.join(ret[0]),
'\nSTDERR:'.join(ret[1]),
)
)
for line in ret[0]:
log.debug('script: salt-minion: stdout: {0}'.format(line))
for line in ret[1]:
log.debug('script: salt-minion: stderr: {0}'.format(line))
if exitstatus is not None:
self.assertEqual(
ret[2],
exitstatus,
'script action "{0}" {1} exited {2}, must be {3}\nSTDOUT:{4}\nSTDERR:{5}'.format(
action,
message,
ret[2],
exitstatus,
'\nSTDOUT:'.join(ret[0]),
'\nSTDERR:'.join(ret[1]),
)
)
return ret
def _initscript_setup(self, minions):
'''Re-usable setup for running salt-minion tests'''
user = getpass.getuser()
_minions = []
for mname in minions:
minion = testprogram.TestDaemonSaltMinion(
name=mname,
config={'user': user},
parent_dir=self._test_dir,
)
# Call setup here to ensure config and script exist
minion.setup()
_minions.append(minion)
# Need salt-call, salt-minion for wrapper script
salt_call = testprogram.TestProgramSaltCall(parent_dir=self._test_dir)
# Ensure that run-time files are generated
salt_call.setup()
sysconf_dir = os.path.dirname(_minions[0].config_dir)
cmd_env = {
'PATH': ':'.join([salt_call.script_dir, os.getenv('PATH')]),
'PYTHONPATH': ':'.join(sys.path),
'SALTMINION_DEBUG': '1' if DEBUG else '',
'SALTMINION_PYTHON': sys.executable,
'SALTMINION_SYSCONFDIR': sysconf_dir,
'SALTMINION_BINDIR': _minions[0].script_dir,
'SALTMINION_CONFIGS': '\n'.join([
'{0} {1}'.format(user, minion.config_dir) for minion in _minions
]),
}
default_dir = os.path.join(sysconf_dir, 'default')
if not os.path.exists(default_dir):
os.makedirs(default_dir)
with open(os.path.join(default_dir, 'salt'), 'w') as defaults:
# Test suites is quite slow - extend the timeout
defaults.write(
'TIMEOUT=60\n'
'TICK=1\n'
)
init_script = testprogram.TestProgram(
name='init:salt-minion',
program=os.path.join(integration.CODE_DIR, 'pkg', 'rpm', 'salt-minion'),
env=cmd_env,
)
return _minions, salt_call, init_script
def test_linux_initscript(self):
'''
Various tests of the init script to verify that it properly controls a salt minion.
'''
pform = platform.uname()[0].lower()
if pform not in ('linux',):
self.skipTest('salt-minion init script is unavailable on {1}'.format(platform))
minions, _, init_script = self._initscript_setup(self._test_minions[:1])
try:
# I take visual readability with aligned columns over strict PEP8
# (bad-whitespace) Exactly one space required after comma
# pylint: disable=C0326
ret = self._run_initscript(init_script, minions, False, 'bogusaction', 2)
ret = self._run_initscript(init_script, minions, False, 'reload', 3) # Not implemented
ret = self._run_initscript(init_script, minions, False, 'stop', 0, 'when not running')
ret = self._run_initscript(init_script, minions, False, 'status', 3, 'when not running')
ret = self._run_initscript(init_script, minions, False, 'condrestart', 7, 'when not running')
ret = self._run_initscript(init_script, minions, False, 'try-restart', 7, 'when not running')
ret = self._run_initscript(init_script, minions, True, 'start', 0, 'when not running')
ret = self._run_initscript(init_script, minions, True, 'status', 0, 'when running')
# Verify that PIDs match
for (minion, stdout) in zip(minions, ret[0]):
status_pid = int(stdout.rsplit(' ', 1)[-1])
self.assertEqual(
status_pid,
minion.daemon_pid,
'PID in "{0}" is {1} and does not match status PID {2}'.format(
minion.pid_path,
minion.daemon_pid,
status_pid
)
)
ret = self._run_initscript(init_script, minions, True, 'start', 0, 'when running')
ret = self._run_initscript(init_script, minions, True, 'condrestart', 0, 'when running')
ret = self._run_initscript(init_script, minions, True, 'try-restart', 0, 'when running')
ret = self._run_initscript(init_script, minions, False, 'stop', 0, 'when running')
finally:
# Ensure that minions are shutdown
for minion in minions:
minion.shutdown()
if __name__ == '__main__':
from integration import run_tests
run_tests(MinionTest)
integration.run_tests(MinionTest)

View file

@ -0,0 +1 @@
# -*- coding: utf-8 -*-

View file

@ -0,0 +1,599 @@
# -*- coding: utf-8 -*-
'''
Classes for starting/stopping/status salt daemons, auxiliary
scripts, generic commands.
'''
import atexit
import copy
from datetime import datetime, timedelta
import logging
import os
import shutil
import signal
import subprocess
import sys
import tempfile
import time
import yaml
import salt.ext.six as six
import salt.utils.process
import salt.utils.psutil_compat as psutils
import integration
LOG = logging.getLogger(__name__)
if 'TimeoutError' not in __builtins__:
class TimeoutError(OSError):
'''Compatibility exception with python3'''
pass
__builtins__['TimeoutError'] = TimeoutError
class TestProgram(object):
'''
Set up an arbitrary executable to run.
'''
def __init__(self, program=None, name=None, env=None, shell=False, parent_dir=None, clean_on_exit=True):
self.program = program or getattr(self, 'program', None)
self.name = name or getattr(self, 'name', None)
self.env = env or {}
self.shell = shell
self._parent_dir = parent_dir or None
self.clean_on_exit = clean_on_exit
if not self.name:
if not self.program:
raise ValueError('"{0}" object must specify "program" parameter'.format(self.__class__.__name__))
self.name = os.path.basename(self.program)
self.process = None
self.created_parent_dir = False
self._setup_done = False
# Register the exit clean-up before making anything needing clean-up
atexit.register(self.cleanup)
def __enter__(self):
pass
def __exit__(self, typ, value, traceback):
pass
@property
def start_pid(self):
'''PID of the called script prior to deamonizing.'''
return self.process.pid if self.process else None
@property
def parent_dir(self):
'''Directory that contains everything generated for running scripts - possibly for multiple scripts.'''
if self._parent_dir is None:
self.created_parent_dir = True
self._parent_dir = tempfile.mkdtemp(prefix='salt-testdaemon-XXXX')
else:
self._parent_dir = os.path.abspath(os.path.normpath(self._parent_dir))
if not os.path.exists(self._parent_dir):
self.created_parent_dir = True
os.makedirs(self._parent_dir)
elif not os.path.isdir(self._parent_dir):
raise ValueError('Parent path "{0}" exists but is not a directory'.format(self._parent_dir))
return self._parent_dir
def setup(self, *args, **kwargs):
'''Create any scaffolding for run-time'''
pass
def cleanup(self, *args, **kwargs):
''' Clean out scaffolding of setup() and any run-time generated files.'''
# Unused for now
_ = args
_ = kwargs
if self.process:
try:
self.process.kill()
self.process.wait()
except OSError:
pass
if self.created_parent_dir and os.path.exists(self.parent_dir):
shutil.rmtree(self.parent_dir)
def run(
self,
args=None,
catch_stderr=False,
with_retcode=False,
timeout=None,
raw=False,
):
'''
Execute a command possibly using a supplied environment.
:param args:
A command string or a command sequence of arguments for the program.
:param catch_stderr: A boolean whether to capture and return stderr.
:param with_retcode: A boolean whether to return the exit code.
:param timeout: A float of how long to wait for the process to
complete before it is killed.
:param raw: A boolean whether to return buffer strings for stdout and
stderr or sequences of output lines.
:param env: A dictionary of environment key/value settings for the
command.
:param shell: A boolean of whether the command is processed by the
shell or invoked with execv.
:return list: (stdout [,stderr] [,retcode])
'''
if not self._setup_done:
self.setup()
self._setup_done = True
if args is None:
args = []
cmd_env = dict(os.environ)
cmd_env.update(self.env)
popen_kwargs = {
'shell': self.shell,
'stdout': subprocess.PIPE,
'env': cmd_env,
}
if catch_stderr is True:
popen_kwargs['stderr'] = subprocess.PIPE
if not sys.platform.lower().startswith('win'):
popen_kwargs['close_fds'] = True
def detach_from_parent_group():
'''
A utility function that prevents child process from getting parent signals.
'''
os.setpgrp()
popen_kwargs['preexec_fn'] = detach_from_parent_group
elif sys.platform.lower().startswith('win') and timeout is not None:
raise RuntimeError('Timeout is not supported under windows')
argv = [self.program]
argv.extend(args)
process = subprocess.Popen(argv, **popen_kwargs)
self.process = process
if timeout is not None:
stop_at = datetime.now() + timedelta(seconds=timeout)
term_sent = False
while True:
process.poll()
if datetime.now() > stop_at:
if term_sent is False:
# Kill the process group since sending the term signal
# would only terminate the shell, not the command
# executed in the shell
os.killpg(os.getpgid(process.pid), signal.SIGINT)
term_sent = True
continue
try:
# As a last resort, kill the process group
os.killpg(os.getpgid(process.pid), signal.SIGKILL)
process.wait()
except OSError as exc:
if exc.errno != 3:
raise
out = process.stdout.read().splitlines()
out.extend([
'Process took more than {0} seconds to complete. '
'Process Killed!'.format(timeout)
])
if catch_stderr:
err = process.stderr.read().splitlines()
if with_retcode:
return out, err, process.returncode
else:
return out, err
if with_retcode:
return out, process.returncode
else:
return out
if process.returncode is not None:
break
if catch_stderr:
if sys.version_info < (2, 7):
# On python 2.6, the subprocess'es communicate() method uses
# select which, is limited by the OS to 1024 file descriptors
# We need more available descriptors to run the tests which
# need the stderr output.
# So instead of .communicate() we wait for the process to
# finish, but, as the python docs state "This will deadlock
# when using stdout=PIPE and/or stderr=PIPE and the child
# process generates enough output to a pipe such that it
# blocks waiting for the OS pipe buffer to accept more data.
# Use communicate() to avoid that." <- a catch, catch situation
#
# Use this work around were it's needed only, python 2.6
process.wait()
out = process.stdout.read()
err = process.stderr.read()
else:
out, err = process.communicate()
# Force closing stderr/stdout to release file descriptors
if process.stdout is not None:
process.stdout.close()
if process.stderr is not None:
process.stderr.close()
# pylint: disable=maybe-no-member
try:
if with_retcode:
if out is not None and err is not None:
if not raw:
return out.splitlines(), err.splitlines(), process.returncode
else:
return out, err, process.returncode
return out.splitlines(), [], process.returncode
else:
if out is not None and err is not None:
if not raw:
return out.splitlines(), err.splitlines()
else:
return out, err
if not raw:
return out.splitlines(), []
else:
return out, []
finally:
try:
process.terminate()
except OSError as err:
# process already terminated
pass
# pylint: enable=maybe-no-member
data = process.communicate()
process.stdout.close()
try:
if with_retcode:
if not raw:
return data[0].splitlines(), process.returncode
else:
return data[0], process.returncode
else:
if not raw:
return data[0].splitlines()
else:
return data[0]
finally:
try:
process.terminate()
except OSError as err:
# process already terminated
pass
class TestSaltProgramMeta(type):
'''
A Meta-class to set self.script from the class name when it is
not specifically set by a "script" argument.
'''
def __new__(mcs, name, bases, attrs):
if attrs.get('script') is None:
if 'Salt' in name:
script = 'salt-{0}'.format(name.rsplit('Salt', 1)[-1].lower())
if script is None:
raise AttributeError(
'Class {0}: Unable to set "script" attribute: class name'
' must include "Salt" or "script" must be explicitly set.'.format(name)
)
attrs['script'] = script
return super(TestSaltProgramMeta, mcs).__new__(mcs, name, bases, attrs)
class TestSaltProgram(TestProgram):
'''
This is like TestProgram but with some functions to run a salt-specific
auxiliary program.
'''
__metaclass__ = TestSaltProgramMeta
script = ''
def __init__(self, *args, **kwargs):
if len(args) < 2 and 'program' not in kwargs:
# This is effectively a place-holder - it gets set correctly after super()
kwargs['program'] = self.script
super(TestSaltProgram, self).__init__(*args, **kwargs)
self.program = self.script_path
@property
def script_dir(self):
'''The directory where the script is written.'''
return os.path.join(self.parent_dir, 'bin')
@property
def script_path(self):
'''Full path of the run-time script.'''
return os.path.join(self.script_dir, self.script)
def setup(self, *args, **kwargs):
super(TestSaltProgram, self).setup(*args, **kwargs)
self.install_script()
def install_script(self):
'''Generate the script file that calls python objects and libraries.'''
if not os.path.exists(self.script_dir):
os.makedirs(self.script_dir)
lines = []
script_source = os.path.join(integration.CODE_DIR, 'scripts', self.script)
with open(script_source, 'r') as sso:
lines.extend(sso.readlines())
if lines[0].startswith('#!'):
lines.pop(0)
lines.insert(0, '#!{0}\n'.format(sys.executable))
with open(self.script_path, 'w') as sdo:
sdo.write(''.join(lines))
sdo.flush()
os.chmod(self.script_path, 0755)
class TestProgramSaltCall(TestSaltProgram):
'''Class to manage salt-call'''
pass
class TestDaemon(TestProgram):
'''
Run one of the standard daemons
'''
script = None
empty_config = ''
pid_file = None
config_file = ''
config_types = (six.string_types,)
dirtree = []
def __init__(self, *args, **kwargs):
self._config = kwargs.pop('config', copy.copy(self.empty_config))
self.script = kwargs.pop('script', self.script)
self.pid_file = kwargs.pop('pid_file', '{0}.pid'.format(self.script))
self.config_file = kwargs.pop('config_file', self.config_file)
if not args and 'program' not in kwargs:
# This is effectively a place-holder - it gets set correctly after super()
kwargs['program'] = self.script
super(TestDaemon, self).__init__(*args, **kwargs)
@property
def root_dir(self):
'''Directory that will contains all of the static and dynamic files for the daemon'''
return os.path.join(self.parent_dir, self.name)
@property
def pid_path(self):
'''Full path of the PID file'''
return os.path.join(self.root_dir, 'var', 'run', self.pid_file)
@property
def daemon_pid(self):
'''Return the daemon PID'''
return (
salt.utils.process.get_pidfile(self.pid_path)
if salt.utils.process.check_pidfile(self.pid_path)
else None
)
def wait_for_daemon_pid(self, timeout=0):
'''Wait up to timeout seconds for the PID file to appear and return the PID'''
endtime = time.time() + timeout
while True:
pid = self.daemon_pid
if pid:
return pid
if endtime < time.time():
raise TimeoutError('Timeout waiting for "{0}" pid in "{1}"'.format(self.name, self.pid_path))
time.sleep(0.5)
def is_running(self):
'''Is the daemon running?'''
ret = False
try:
pid = self.wait_for_daemon_pid()
ret = psutils.pid_exists(pid)
except TimeoutError:
pass
return ret
def shutdown(self, signum=signal.SIGTERM, timeout=10):
'''Shutdown a running daemon'''
if self.process:
pid = self.wait_for_daemon_pid()
os.kill(pid, signum)
salt.utils.process.clean_proc(pid, wait_for_kill=timeout)
@property
def config_dir(self):
'''Directory of the config file'''
return os.path.join(self.root_dir, 'etc', 'salt')
def setup(self, *args, **kwargs):
'''Perform any necessary setup to be ready to run'''
super(TestDaemon, self).setup(*args, **kwargs)
self.config_write()
self.make_dirtree()
def cleanup(self, *args, **kwargs):
'''Remove left-over scaffolding - antithesis of setup()'''
self.shutdown()
if os.path.exists(self.root_dir):
shutil.rmtree(self.root_dir)
super(TestDaemon, self).cleanup(*args, **kwargs)
def config_write(self):
'''Write out the config to a file'''
if not os.path.exists(self.config_dir):
os.makedirs(self.config_dir)
config_path = os.path.join(self.config_dir, self.config_file)
with open(config_path, 'w') as cfo:
cfo.write(self.config_stringify())
cfo.flush()
def make_dirtree(self):
'''Create directory structure.'''
for branch in self.dirtree:
path = os.path.join(self.root_dir, branch)
if not os.path.exists(path):
os.makedirs(path)
def config_type(self, config):
'''Check if a configuration is an acceptable type.'''
return isinstance(config, self.config_types)
def config_cast(self, config):
'''Cast a configuration to the internal expected type.'''
if not isinstance(config, six.string_types):
config = str(config)
return config
def config_stringify(self):
'''Marshall the configuration to a string'''
return self.config
def config_merge(self, base, overrides):
'''Merge two configuration hunks'''
base = self.config_cast(base)
overrides = self.config_cast(overrides)
return ''.join([base, overrides])
@property
def config(self):
'''Get the configuration'''
return self._config
@config.setter
def config(self, val):
'''Set the configuration'''
if val is None:
val = ''
self._config = self.config_cast(val)
class TestSaltDaemonMeta(TestSaltProgramMeta, type):
'''
A meta-class to stack all inherited config_attrs from the base classes.
'''
def __new__(mcs, name, bases, attrs):
config_attrs = {}
dirtree = set()
for base in bases:
config_attrs.update(getattr(base, 'config_attrs', {}))
dirtree.update(getattr(base, 'dirtree', []))
config_attrs.update(attrs.get('config_attrs', {}))
dirtree.update(attrs.get('dirtree', []))
attrs['config_attrs'] = config_attrs
attrs['dirtree'] = dirtree
return super(TestSaltDaemonMeta, mcs).__new__(mcs, name, bases, attrs)
class TestSaltDaemon(TestDaemon, TestSaltProgram):
'''
A class to run arbitrary salt daemons (master, minion, syndic, etc.)
'''
__metaclass__ = TestSaltDaemonMeta
config_types = (dict,)
config_attrs = {
'root_dir': None,
'config_dir': None,
}
script = ''
empty_config = {}
dirtree = [
'var/log/salt',
]
def __init__(self, *args, **kwargs):
super(TestSaltDaemon, self).__init__(*args, **kwargs)
path = self.env.get('PATH', os.getenv('PATH'))
self.env['PATH'] = ':'.join([self.script_dir, path])
def config_cast(self, config):
if isinstance(config, six.string_types):
config = yaml.safe_load(config)
return config
def config_merge(self, base, overrides):
_base = self.config_cast(copy.deepcopy(base))
_overrides = self.config_cast(overrides)
# NOTE: this simple update will not work for deep dictionaries
_base.update(copy.deepcopy(_overrides))
return _base
@property
def config(self):
attr_vals = dict([(k, getattr(self, v if v else k)) for k, v in self.config_attrs.items()])
merged = self.config_merge(self._config, attr_vals)
return merged
def config_stringify(self):
return yaml.safe_dump(self.config, default_flow_style=False)
class TestDaemonSaltMaster(TestSaltDaemon):
'''
Manager for salt-master daemon.
'''
config_file = 'master'
class TestDaemonSaltMinion(TestSaltDaemon):
'''
Manager for salt-minion daemon.
'''
config_attrs = {
'id': 'name',
}
config_file = 'minion'
class TestDaemonSaltApi(TestSaltDaemon):
'''
Manager for salt-api daemon.
'''
pass
class TestDaemonSaltSyndic(TestSaltDaemon):
'''
Manager for salt-syndic daemon.
'''
pass