Using pytest start_daemon

tests/integration/__init__.py

@@ -34,12 +34,19 @@ STATE_FUNCTION_RUNNING_RE = re.compile(
     r'''The function (?:"|')(?P<state_func>.*)(?:"|') is running as PID '''
     r'(?P<pid>[\d]+) and was started at (?P<date>.*) with jid (?P<jid>[\d]+)'
 )
-INTEGRATION_TEST_DIR = os.path.dirname(
-    os.path.normpath(os.path.abspath(__file__))
-)
+
+TESTS_DIR = os.path.dirname(os.path.dirname(os.path.normpath(os.path.abspath(__file__))))
 if os.name == 'nt':
-    INTEGRATION_TEST_DIR = INTEGRATION_TEST_DIR.replace('\\', '\\\\')
-CODE_DIR = os.path.dirname(os.path.dirname(INTEGRATION_TEST_DIR))
+    TESTS_DIR = TESTS_DIR.replace('\\', '\\\\')
+CODE_DIR = os.path.dirname(TESTS_DIR)
+
+# Let's inject CODE_DIR so salt is importable if not there already
+if CODE_DIR not in sys.path:
+    sys.path.insert(0, CODE_DIR)
+
+# Import salt tests support dirs
+from tests.support.paths import *  # pylint: disable=wildcard-import
+from tests.support.processes import *  # pylint: disable=wildcard-import
 
 # Import Salt Testing libs
 from salttesting import TestCase
@@ -49,9 +56,6 @@ from salttesting.parser import PNUM, print_header, SaltTestcaseParser
 from salttesting.helpers import requires_sshd_server
 from salttesting.helpers import ensure_in_syspath, RedirectStdStreams
 
-# Update sys.path
-ensure_in_syspath(CODE_DIR)
-
 # Import Salt libs
 import salt
 import salt.config
@@ -103,6 +107,8 @@ if salt.utils.is_windows():
 from tornado import gen
 from tornado import ioloop
 
+# Import salt tests support libs
+from tests.support.processes import SaltMaster, SaltMinion, SaltSyndic
 try:
     from salttesting.helpers import terminate_process_pid
 except ImportError:
@@ -190,57 +196,6 @@ except ImportError:
             if children:
                 psutil.wait_procs(children, timeout=1, callback=lambda proc: kill_children(children, kill=True))
 
-
-SYS_TMP_DIR = os.path.realpath(
-    # Avoid ${TMPDIR} and gettempdir() on MacOS as they yield a base path too long
-    # for unix sockets: ``error: AF_UNIX path too long``
-    # Gentoo Portage prefers ebuild tests are rooted in ${TMPDIR}
-    os.environ.get('TMPDIR', tempfile.gettempdir()) if not salt.utils.is_darwin() else '/tmp'
-)
-TMP = os.path.join(SYS_TMP_DIR, 'salt-tests-tmpdir')
-FILES = os.path.join(INTEGRATION_TEST_DIR, 'files')
-PYEXEC = 'python{0}.{1}'.format(*sys.version_info)
-MOCKBIN = os.path.join(INTEGRATION_TEST_DIR, 'mockbin')
-SCRIPT_DIR = os.path.join(CODE_DIR, 'scripts')
-TMP_STATE_TREE = os.path.join(SYS_TMP_DIR, 'salt-temp-state-tree')
-TMP_PRODENV_STATE_TREE = os.path.join(SYS_TMP_DIR, 'salt-temp-prodenv-state-tree')
-TMP_CONF_DIR = os.path.join(TMP, 'config')
-TMP_SUB_MINION_CONF_DIR = os.path.join(TMP_CONF_DIR, 'sub-minion')
-TMP_SYNDIC_MINION_CONF_DIR = os.path.join(TMP_CONF_DIR, 'syndic-minion')
-TMP_SYNDIC_MASTER_CONF_DIR = os.path.join(TMP_CONF_DIR, 'syndic-master')
-CONF_DIR = os.path.join(INTEGRATION_TEST_DIR, 'files', 'conf')
-PILLAR_DIR = os.path.join(FILES, 'pillar')
-TMP_SCRIPT_DIR = os.path.join(TMP, 'scripts')
-ENGINES_DIR = os.path.join(FILES, 'engines')
-LOG_HANDLERS_DIR = os.path.join(FILES, 'log_handlers')
-
-SCRIPT_TEMPLATES = {
-    'salt': [
-        'from salt.scripts import salt_main\n',
-        'if __name__ == \'__main__\':\n'
-        '    salt_main()'
-    ],
-    'salt-api': [
-        'import salt.cli\n',
-        'def main():\n',
-        '    sapi = salt.cli.SaltAPI()',
-        '    sapi.start()\n',
-        'if __name__ == \'__main__\':',
-        '    main()'
-    ],
-    'common': [
-        'from salt.scripts import salt_{0}\n',
-        'from salt.utils import is_windows\n\n',
-        'if __name__ == \'__main__\':\n',
-        '    if is_windows():\n',
-        '        import os.path\n',
-        '        import py_compile\n',
-        '        cfile = os.path.splitext(__file__)[0] + ".pyc"\n',
-        '        if not os.path.exists(cfile):\n',
-        '            py_compile.compile(__file__, cfile)\n',
-        '    salt_{0}()'
-    ]
-}
 RUNTIME_CONFIGS = {}
 
 log = logging.getLogger(__name__)
@@ -409,273 +364,6 @@ class SocketServerRequestHandler(socketserver.StreamRequestHandler):
                 log.exception(exc)
 
 
-class ScriptPathMixin(object):
-
-    def get_script_path(self, script_name):
-        '''
-        Return the path to a testing runtime script
-        '''
-        if not os.path.isdir(TMP_SCRIPT_DIR):
-            os.makedirs(TMP_SCRIPT_DIR)
-
-        script_path = os.path.join(TMP_SCRIPT_DIR,
-                                   'cli_{0}.py'.format(script_name.replace('-', '_')))
-
-        if not os.path.isfile(script_path):
-            log.info('Generating {0}'.format(script_path))
-
-            # Late import
-            import salt.utils
-
-            with salt.utils.fopen(script_path, 'w') as sfh:
-                script_template = SCRIPT_TEMPLATES.get(script_name, None)
-                if script_template is None:
-                    script_template = SCRIPT_TEMPLATES.get('common', None)
-                if script_template is None:
-                    raise RuntimeError(
-                        '{0} does not know how to handle the {1} script'.format(
-                            self.__class__.__name__,
-                            script_name
-                        )
-                    )
-                sfh.write(
-                    '#!{0}\n\n'.format(sys.executable) +
-                    'import sys\n' +
-                    'CODE_DIR="{0}"\n'.format(CODE_DIR) +
-                    'if CODE_DIR not in sys.path:\n' +
-                    '    sys.path.insert(0, CODE_DIR)\n\n' +
-                    '\n'.join(script_template).format(script_name.replace('salt-', ''))
-                )
-            fst = os.stat(script_path)
-            os.chmod(script_path, fst.st_mode | stat.S_IEXEC)
-
-        log.info('Returning script path %r', script_path)
-        return script_path
-
-
-class SaltScriptBase(ScriptPathMixin):
-    '''
-    Base class for Salt CLI scripts
-    '''
-
-    cli_script_name = None
-
-    def __init__(self,
-                 config,
-                 config_dir,
-                 bin_dir_path,
-                 io_loop=None):
-        self.config = config
-        self.config_dir = config_dir
-        self.bin_dir_path = bin_dir_path
-        self._io_loop = io_loop
-
-    @property
-    def io_loop(self):
-        '''
-        Return an IOLoop
-        '''
-        if self._io_loop is None:
-            self._io_loop = ioloop.IOLoop.current()
-        return self._io_loop
-
-    def get_script_args(self):  # pylint: disable=no-self-use
-        '''
-        Returns any additional arguments to pass to the CLI script
-        '''
-        return []
-
-
-class SaltDaemonScriptBase(SaltScriptBase, ShellTestCase):
-    '''
-    Base class for Salt Daemon CLI scripts
-    '''
-
-    def __init__(self, *args, **kwargs):
-        super(SaltDaemonScriptBase, self).__init__(*args, **kwargs)
-        self._running = multiprocessing.Event()
-        self._connectable = multiprocessing.Event()
-        self._process = None
-
-    def is_alive(self):
-        '''
-        Returns true if the process is alive
-        '''
-        return self._running.is_set()
-
-    def get_check_ports(self):  # pylint: disable=no-self-use
-        '''
-        Return a list of ports to check against to ensure the daemon is running
-        '''
-        return []
-
-    def start(self):
-        '''
-        Start the daemon subprocess
-        '''
-        self._process = salt.utils.process.SignalHandlingMultiprocessingProcess(
-            target=self._start, args=(self._running,))
-        self._process.start()
-        self._running.set()
-        return True
-
-    def _start(self, running_event):
-        '''
-        The actual, coroutine aware, start method
-        '''
-        log.info('Starting %s %s DAEMON', self.display_name, self.__class__.__name__)
-        proc_args = [
-            self.get_script_path(self.cli_script_name),
-            '-c',
-            self.config_dir,
-        ] + self.get_script_args()
-        if salt.utils.is_windows():
-            # Windows need the python executable to come first
-            proc_args.insert(0, sys.executable)
-        log.info('Running \'%s\' from %s...', ' '.join(proc_args), self.__class__.__name__)
-
-        try:
-            terminal = NonBlockingPopen(proc_args, cwd=CODE_DIR)
-
-            while running_event.is_set() and terminal.poll() is None:
-                # We're not actually interested in processing the output, just consume it
-                if terminal.stdout is not None:
-                    terminal.recv()
-                if terminal.stderr is not None:
-                    terminal.recv_err()
-                time.sleep(0.125)
-        except (SystemExit, KeyboardInterrupt):
-            pass
-
-        terminate_process_pid(terminal.pid)
-#        terminal.communicate()
-
-    def terminate(self):
-        '''
-        Terminate the started daemon
-        '''
-        log.info('Terminating %s %s DAEMON', self.display_name, self.__class__.__name__)
-        self._running.clear()
-        self._connectable.clear()
-        time.sleep(0.0125)
-        terminate_process_pid(self._process.pid)
-#        self._process.join()
-        log.info('%s %s DAEMON terminated', self.display_name, self.__class__.__name__)
-
-    def wait_until_running(self, timeout=None):
-        '''
-        Blocking call to wait for the daemon to start listening
-        '''
-        if self._connectable.is_set():
-            return True
-        try:
-            return self.io_loop.run_sync(self._wait_until_running, timeout=timeout)
-        except ioloop.TimeoutError:
-            return False
-
-    @gen.coroutine
-    def _wait_until_running(self):
-        '''
-        The actual, coroutine aware, call to wait for the daemon to start listening
-        '''
-        check_ports = self.get_check_ports()
-        log.debug(
-            '%s is checking the following ports to assure running status: %s',
-            self.__class__.__name__,
-            check_ports
-        )
-        while self._running.is_set():
-            if not check_ports:
-                self._connectable.set()
-                break
-            for port in set(check_ports):
-                if isinstance(port, int):
-                    log.trace('Checking connectable status on port: %s', port)
-                    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-                    conn = sock.connect_ex(('localhost', port))
-                    if conn == 0:
-                        log.debug('Port %s is connectable!', port)
-                        check_ports.remove(port)
-                        try:
-                            sock.shutdown(socket.SHUT_RDWR)
-                            sock.close()
-                        except socket.error as exc:
-                            if not sys.platform.startswith('darwin'):
-                                raise
-                            try:
-                                if exc.errno != errno.ENOTCONN:
-                                    raise
-                            except AttributeError:
-                                # This is not macOS !?
-                                pass
-                    del sock
-                elif isinstance(port, str):
-                    joined = self.run_run('manage.joined', config_dir=self.config_dir)
-                    joined = [x.lstrip('- ') for x in joined]
-                    if port in joined:
-                        check_ports.remove(port)
-            yield gen.sleep(0.125)
-        # A final sleep to allow the ioloop to do other things
-        yield gen.sleep(0.125)
-        log.info('All ports checked. %s running!', self.cli_script_name)
-        raise gen.Return(self._connectable.is_set())
-
-
-class SaltMinion(SaltDaemonScriptBase):
-    '''
-    Class which runs the salt-minion daemon
-    '''
-
-    cli_script_name = 'salt-minion'
-
-    def get_script_args(self):
-        script_args = ['-l', 'quiet']
-        if salt.utils.is_windows() is False:
-            script_args.append('--disable-keepalive')
-        return script_args
-
-    def get_check_ports(self):
-        if salt.utils.is_windows():
-            return set([self.config['tcp_pub_port'],
-                        self.config['tcp_pull_port']])
-        else:
-            return set([self.config['id']])
-
-
-class SaltMaster(SaltDaemonScriptBase):
-    '''
-    Class which runs the salt-minion daemon
-    '''
-
-    cli_script_name = 'salt-master'
-
-    def get_check_ports(self):
-        #return set([self.config['runtests_conn_check_port']])
-        return set([self.config['ret_port'],
-                    self.config['publish_port']])
-        # Disabled along with Pytest config until fixed.
-#                    self.config['runtests_conn_check_port']])
-
-    def get_script_args(self):
-        #return ['-l', 'debug']
-        return ['-l', 'quiet']
-
-
-class SaltSyndic(SaltDaemonScriptBase):
-    '''
-    Class which runs the salt-syndic daemon
-    '''
-
-    cli_script_name = 'salt-syndic'
-
-    def get_script_args(self):
-        #return ['-l', 'debug']
-        return ['-l', 'quiet']
-
-    def get_check_ports(self):
-        return set()
-
-
 class TestDaemon(object):
     '''
     Set up the master and minion daemons, and run related cases
@@ -787,39 +475,184 @@ class TestDaemon(object):
         self.log_server_process = threading.Thread(target=self.log_server.serve_forever)
         self.log_server_process.daemon = True
         self.log_server_process.start()
-
-        self.master_process = SaltMaster(self.master_opts, TMP_CONF_DIR, SCRIPT_DIR)
-        self.master_process.display_name = 'salt-master'
-        self.minion_process = SaltMinion(self.minion_opts, TMP_CONF_DIR, SCRIPT_DIR)
-        self.minion_process.display_name = 'salt-minion'
-        self.sub_minion_process = SaltMinion(self.sub_minion_opts, TMP_SUB_MINION_CONF_DIR, SCRIPT_DIR)
-        self.sub_minion_process.display_name = 'sub salt-minion'
-        self.smaster_process = SaltMaster(self.syndic_master_opts, TMP_SYNDIC_MASTER_CONF_DIR, SCRIPT_DIR)
-        self.smaster_process.display_name = 'syndic salt-master'
-        self.syndic_process = SaltSyndic(self.syndic_opts, TMP_SYNDIC_MINION_CONF_DIR, SCRIPT_DIR)
-        self.syndic_process.display_name = 'salt-syndic'
-        for process in (self.master_process, self.minion_process, self.sub_minion_process,
-                        self.smaster_process, self.syndic_process):
+        try:
             sys.stdout.write(
-                ' * {LIGHT_YELLOW}Starting {0} ... {ENDC}'.format(
-                    process.display_name,
-                    **self.colors
-                )
+                ' * {LIGHT_YELLOW}Starting salt-master ... {ENDC}'.format(**self.colors)
             )
             sys.stdout.flush()
-            process.start()
-            process.wait_until_running(timeout=60)
+            self.master_process = start_daemon(
+                daemon_name='salt-master',
+                daemon_id=self.master_opts['id'],
+                daemon_log_prefix='salt-master/{}'.format(self.master_opts['id']),
+                daemon_cli_script_name='master',
+                daemon_config=self.master_opts,
+                daemon_config_dir=TMP_CONF_DIR,
+                daemon_class=SaltMaster,
+                bin_dir_path=SCRIPT_DIR,
+                fail_hard=True,
+                start_timeout=30)
             sys.stdout.write(
                 '\r{0}\r'.format(
                     ' ' * getattr(self.parser.options, 'output_columns', PNUM)
                 )
             )
             sys.stdout.write(
-                ' * {LIGHT_GREEN}Starting {0} ... STARTED!\n{ENDC}'.format(
-                    process.display_name,
-                    **self.colors
+                ' * {LIGHT_GREEN}Starting salt-master ... STARTED!\n{ENDC}'.format(**self.colors)
+            )
+            sys.stdout.flush()
+        except (RuntimeWarning, RuntimeError):
+            sys.stdout.write(
+                '\r{0}\r'.format(
+                    ' ' * getattr(self.parser.options, 'output_columns', PNUM)
                 )
             )
+            sys.stdout.write(
+                ' * {LIGHT_RED}Starting salt-master ... FAILED!\n{ENDC}'.format(**self.colors)
+            )
+            sys.stdout.flush()
+
+        try:
+            sys.stdout.write(
+                ' * {LIGHT_YELLOW}Starting salt-minion ... {ENDC}'.format(**self.colors)
+            )
+            sys.stdout.flush()
+            self.minion_process = start_daemon(
+                daemon_name='salt-minion',
+                daemon_id=self.master_opts['id'],
+                daemon_log_prefix='salt-minion/{}'.format(self.minion_opts['id']),
+                daemon_cli_script_name='minion',
+                daemon_config=self.minion_opts,
+                daemon_config_dir=TMP_CONF_DIR,
+                daemon_class=SaltMinion,
+                bin_dir_path=SCRIPT_DIR,
+                fail_hard=True,
+                start_timeout=30)
+            sys.stdout.write(
+                '\r{0}\r'.format(
+                    ' ' * getattr(self.parser.options, 'output_columns', PNUM)
+                )
+            )
+            sys.stdout.write(
+                ' * {LIGHT_GREEN}Starting salt-minion ... STARTED!\n{ENDC}'.format(**self.colors)
+            )
+            sys.stdout.flush()
+        except (RuntimeWarning, RuntimeError):
+            sys.stdout.write(
+                '\r{0}\r'.format(
+                    ' ' * getattr(self.parser.options, 'output_columns', PNUM)
+                )
+            )
+            sys.stdout.write(
+                ' * {LIGHT_RED}Starting salt-minion ... FAILED!\n{ENDC}'.format(**self.colors)
+            )
+            sys.stdout.flush()
+
+        try:
+            sys.stdout.write(
+                ' * {LIGHT_YELLOW}Starting sub salt-minion ... {ENDC}'.format(**self.colors)
+            )
+            sys.stdout.flush()
+            self.sub_minion_process = start_daemon(
+                daemon_name='sub salt-minion',
+                daemon_id=self.master_opts['id'],
+                daemon_log_prefix='sub-salt-minion/{}'.format(self.sub_minion_opts['id']),
+                daemon_cli_script_name='minion',
+                daemon_config=self.sub_minion_opts,
+                daemon_config_dir=TMP_SUB_MINION_CONF_DIR,
+                daemon_class=SaltMinion,
+                bin_dir_path=SCRIPT_DIR,
+                fail_hard=True,
+                start_timeout=30)
+            sys.stdout.write(
+                '\r{0}\r'.format(
+                    ' ' * getattr(self.parser.options, 'output_columns', PNUM)
+                )
+            )
+            sys.stdout.write(
+                ' * {LIGHT_GREEN}Starting sub salt-minion ... STARTED!\n{ENDC}'.format(**self.colors)
+            )
+            sys.stdout.flush()
+        except (RuntimeWarning, RuntimeError):
+            sys.stdout.write(
+                '\r{0}\r'.format(
+                    ' ' * getattr(self.parser.options, 'output_columns', PNUM)
+                )
+            )
+            sys.stdout.write(
+                ' * {LIGHT_RED}Starting sub salt-minion ... FAILED!\n{ENDC}'.format(**self.colors)
+            )
+            sys.stdout.flush()
+
+        try:
+            sys.stdout.write(
+                ' * {LIGHT_YELLOW}Starting syndic salt-master ... {ENDC}'.format(**self.colors)
+            )
+            sys.stdout.flush()
+            self.smaster_process = start_daemon(
+                daemon_name='salt-smaster',
+                daemon_id=self.syndic_master_opts['id'],
+                daemon_log_prefix='salt-smaster/{}'.format(self.syndic_master_opts['id']),
+                daemon_cli_script_name='master',
+                daemon_config=self.syndic_master_opts,
+                daemon_config_dir=TMP_SYNDIC_MASTER_CONF_DIR,
+                daemon_class=SaltMaster,
+                bin_dir_path=SCRIPT_DIR,
+                fail_hard=True,
+                start_timeout=30)
+            sys.stdout.write(
+                '\r{0}\r'.format(
+                    ' ' * getattr(self.parser.options, 'output_columns', PNUM)
+                )
+            )
+            sys.stdout.write(
+                ' * {LIGHT_GREEN}Starting syndic salt-master ... STARTED!\n{ENDC}'.format(**self.colors)
+            )
+            sys.stdout.flush()
+        except (RuntimeWarning, RuntimeError):
+            sys.stdout.write(
+                '\r{0}\r'.format(
+                    ' ' * getattr(self.parser.options, 'output_columns', PNUM)
+                )
+            )
+            sys.stdout.write(
+                ' * {LIGHT_RED}Starting syndic salt-master ... FAILED!\n{ENDC}'.format(**self.colors)
+            )
+            sys.stdout.flush()
+
+        try:
+            sys.stdout.write(
+                ' * {LIGHT_YELLOW}Starting salt-syndic ... {ENDC}'.format(**self.colors)
+            )
+            sys.stdout.flush()
+            self.syndic_process = start_daemon(
+                daemon_name='salt-syndic',
+                daemon_id=self.syndic_opts['id'],
+                daemon_log_prefix='salt-syndic/{}'.format(self.syndic_opts['id']),
+                daemon_cli_script_name='syndic',
+                daemon_config=self.syndic_opts,
+                daemon_config_dir=TMP_SYNDIC_MINION_CONF_DIR,
+                daemon_class=SaltSyndic,
+                bin_dir_path=SCRIPT_DIR,
+                fail_hard=True,
+                start_timeout=30)
+            sys.stdout.write(
+                '\r{0}\r'.format(
+                    ' ' * getattr(self.parser.options, 'output_columns', PNUM)
+                )
+            )
+            sys.stdout.write(
+                ' * {LIGHT_GREEN}Starting salt-syndic ... STARTED!\n{ENDC}'.format(**self.colors)
+            )
+            sys.stdout.flush()
+        except (RuntimeWarning, RuntimeError):
+            sys.stdout.write(
+                '\r{0}\r'.format(
+                    ' ' * getattr(self.parser.options, 'output_columns', PNUM)
+                )
+            )
+            sys.stdout.write(
+                ' * {LIGHT_RED}Starting salt-syndic ... FAILED!\n{ENDC}'.format(**self.colors)
+            )
             sys.stdout.flush()
 
     def start_raet_daemons(self):
@@ -1185,7 +1018,22 @@ class TestDaemon(object):
             syndic_opts[optname] = optname_path
             syndic_master_opts[optname] = optname_path
 
+        master_opts['runtests_conn_check_port'] = get_unused_localhost_port()
+        minion_opts['runtests_conn_check_port'] = get_unused_localhost_port()
+        sub_minion_opts['runtests_conn_check_port'] = get_unused_localhost_port()
+        syndic_opts['runtests_conn_check_port'] = get_unused_localhost_port()
+        syndic_master_opts['runtests_conn_check_port'] = get_unused_localhost_port()
+
         for conf in (master_opts, minion_opts, sub_minion_opts, syndic_opts, syndic_master_opts):
+            if 'engines' not in conf:
+                conf['engines'] = []
+            conf['engines'].append({'salt_runtests': {}})
+
+            if 'engines_dirs' not in conf:
+                conf['engines_dirs'] = []
+
+            conf['engines_dirs'].insert(0, ENGINES_DIR)
+
             if 'log_handlers_dirs' not in conf:
                 conf['log_handlers_dirs'] = []
             conf['log_handlers_dirs'].insert(0, LOG_HANDLERS_DIR)

tests/integration/files/engines/runtests_engine.py

@@ -37,24 +37,27 @@ def __virtual__():
 
 
 def start():
-    # Create our own IOLoop, we're in another process
-    io_loop = ioloop.IOLoop()
-    io_loop.make_current()
-    pytest_engine = PyTestEngine(__opts__, io_loop)  # pylint: disable=undefined-variable
-    io_loop.add_callback(pytest_engine.start)
-    io_loop.start()
+    pytest_engine = PyTestEngine(__opts__)  # pylint: disable=undefined-variable
+    pytest_engine.start()
 
 
 class PyTestEngine(object):
-    def __init__(self, opts, io_loop):
+    def __init__(self, opts):
         self.opts = opts
-        self.io_loop = io_loop
         self.sock = None
 
-    @gen.coroutine
     def start(self):
+        self.io_loop = ioloop.IOLoop()
+        self.io_loop.make_current()
+        self.io_loop.add_callback(self._start)
+        self.io_loop.start()
+
+    @gen.coroutine
+    def _start(self):
         if self.opts['__role'] == 'minion':
             yield self.listen_to_minion_connected_event()
+        else:
+            self.io_loop.spawn_callback(self.fire_master_started_event)
 
         port = int(self.opts['runtests_conn_check_port'])
         log.info('Starting Pytest Engine(role=%s) on port %s', self.opts['__role'], port)
@@ -91,9 +94,7 @@ class PyTestEngine(object):
     def listen_to_minion_connected_event(self):
         log.info('Listening for minion connected event...')
         minion_start_event_match = 'salt/minion/{0}/start'.format(self.opts['id'])
-        event_bus = salt.utils.event.get_master_event(self.opts,
-                                                      self.opts['sock_dir'],
-                                                      listen=True)
+        event_bus = salt.utils.event.get_master_event(self.opts, self.opts['sock_dir'], listen=True)
         event_bus.subscribe(minion_start_event_match)
         while True:
             event = event_bus.get_event(full=True, no_block=True)
@@ -101,3 +102,19 @@ class PyTestEngine(object):
                 log.info('Got minion connected event: %s', event)
                 break
             yield gen.sleep(0.25)
+
+    @gen.coroutine
+    def fire_master_started_event(self):
+        log.info('Firing salt-master started event...')
+        event_bus = salt.utils.event.get_master_event(self.opts, self.opts['sock_dir'], listen=False)
+        master_start_event_tag = 'salt/master/{0}/start'.format(self.opts['id'])
+        load = {'id': self.opts['id'], 'tag': master_start_event_tag, 'data': {}}
+        # One minute should be more than enough to fire these events every second in order
+        # for pytest-salt to pickup that the master is running
+        timeout = 60
+        while True:
+            timeout -= 1
+            event_bus.fire_event(load, master_start_event_tag, timeout=500)
+            if timeout <= 0:
+                break
+            yield gen.sleep(1)

tests/runtests.py

@@ -8,13 +8,59 @@ Discover all instances of unittest.TestCase in this directory.
 # Import python libs
 from __future__ import absolute_import, print_function
 import os
+import imp
 import sys
 import time
 
+TESTS_DIR = os.path.dirname(os.path.normpath(os.path.abspath(__file__)))
+if os.name == 'nt':
+    TESTS_DIR = TESTS_DIR.replace('\\', '\\\\')
+CODE_DIR = os.path.dirname(TESTS_DIR)
+os.chdir(CODE_DIR)
+
+try:
+    import tests
+    if not tests.__file__.startswith(CODE_DIR):
+        print('Found tests module not from salt in {}'.format(tests.__file__))
+        sys.modules.pop('tests')
+        module_dir = os.path.dirname(tests.__file__)
+        if module_dir in sys.path:
+            sys.path.remove(module_dir)
+        del tests
+except ImportError:
+    pass
+
+#tests = imp.load_source('tests', os.path.join(TESTS_DIR, '__init__.py'))
+#salt = imp.load_source('salt', os.path.join(CODE_DIR, 'salt', '__init__.py'))
+
+# Let's inject CODE_DIR so salt is importable if not there already
+if TESTS_DIR in sys.path:
+    sys.path.remove(TESTS_DIR)
+if CODE_DIR in sys.path and sys.path[0] != CODE_DIR:
+    sys.path.remove(CODE_DIR)
+if CODE_DIR not in sys.path:
+    sys.path.insert(0, CODE_DIR)
+if TESTS_DIR not in sys.path:
+    sys.path.insert(1, TESTS_DIR)
+
 # Import salt libs
-from integration import TestDaemon, TMP  # pylint: disable=W0403
-from integration import SYS_TMP_DIR, INTEGRATION_TEST_DIR
-from integration import CODE_DIR as SALT_ROOT
+try:
+    from tests.support.paths import TMP, SYS_TMP_DIR, INTEGRATION_TEST_DIR
+    from tests.support.paths import CODE_DIR as SALT_ROOT
+except ImportError as exc:
+    try:
+        import tests
+        print('Found tests module not from salt in {}'.format(tests.__file__))
+    except ImportError:
+        print('Unable to import salt test module')
+        print(os.environ.get('PYTHONPATH'))
+        pass
+    print('Current sys.paths')
+    import pprint
+    pprint.pprint(sys.path)
+    raise exc
+
+from tests.integration import TestDaemon  # pylint: disable=W0403
 import salt.utils
 
 if not salt.utils.is_windows():
@@ -23,12 +69,9 @@ if not salt.utils.is_windows():
 # Import Salt Testing libs
 from salttesting.parser import PNUM, print_header
 from salttesting.parser.cover import SaltCoverageTestingParser
-try:
-    from salttesting.helpers import terminate_process_pid
-    RUNTESTS_WITH_HARD_KILL = True
-except ImportError:
-    from integration import terminate_process_pid
-    RUNTESTS_WITH_HARD_KILL = False
+
+# Import test support libs
+from tests.support.processes import collect_child_processes, terminate_process
 
 XML_OUTPUT_DIR = os.environ.get(
     'SALT_XML_TEST_REPORTS_DIR',
@@ -610,9 +653,9 @@ class SaltTestsuiteParser(SaltCoverageTestingParser):
         return status
 
     def print_overall_testsuite_report(self):
-        if RUNTESTS_WITH_HARD_KILL is False:
-            terminate_process_pid(os.getpid(), only_children=True)
+        children = collect_child_processes(os.getpid())
         SaltCoverageTestingParser.print_overall_testsuite_report(self)
+        terminate_process(children=children)
 
 
 def main():

tests/support/paths.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+'''
+    :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)`
+    :copyright: © 2017 by the SaltStack Team, see AUTHORS for more details.
+    :license: Apache 2.0, see LICENSE for more details.
+
+
+    tests.support.paths
+    ~~~~~~~~~~~~~~~~~~~
+
+    Tests related paths
+'''
+
+# Import python libs
+from __future__ import absolute_import
+import os
+import sys
+import stat
+import logging
+import tempfile
+
+log = logging.getLogger(__name__)
+
+TESTS_DIR = os.path.dirname(os.path.dirname(os.path.normpath(os.path.abspath(__file__))))
+if os.name == 'nt':
+    TESTS_DIR = TESTS_DIR.replace('\\', '\\\\')
+CODE_DIR = os.path.dirname(TESTS_DIR)
+INTEGRATION_TEST_DIR = os.path.join(TESTS_DIR, 'integration')
+
+# Let's inject CODE_DIR so salt is importable if not there already
+if TESTS_DIR in sys.path:
+    sys.path.remove(TESTS_DIR)
+if CODE_DIR in sys.path and sys.path[0] != CODE_DIR:
+    sys.path.remove(CODE_DIR)
+if CODE_DIR not in sys.path:
+    sys.path.insert(0, CODE_DIR)
+if TESTS_DIR not in sys.path:
+    sys.path.insert(1, TESTS_DIR)
+
+SYS_TMP_DIR = os.path.realpath(
+    # Avoid ${TMPDIR} and gettempdir() on MacOS as they yield a base path too long
+    # for unix sockets: ``error: AF_UNIX path too long``
+    # Gentoo Portage prefers ebuild tests are rooted in ${TMPDIR}
+    os.environ.get('TMPDIR', tempfile.gettempdir()) if not sys.platform.startswith('darwin') else '/tmp'
+)
+TMP = os.path.join(SYS_TMP_DIR, 'salt-tests-tmpdir')
+FILES = os.path.join(INTEGRATION_TEST_DIR, 'files')
+PYEXEC = 'python{0}.{1}'.format(*sys.version_info)
+MOCKBIN = os.path.join(INTEGRATION_TEST_DIR, 'mockbin')
+SCRIPT_DIR = os.path.join(CODE_DIR, 'scripts')
+TMP_STATE_TREE = os.path.join(SYS_TMP_DIR, 'salt-temp-state-tree')
+TMP_PRODENV_STATE_TREE = os.path.join(SYS_TMP_DIR, 'salt-temp-prodenv-state-tree')
+TMP_CONF_DIR = os.path.join(TMP, 'config')
+TMP_SUB_MINION_CONF_DIR = os.path.join(TMP_CONF_DIR, 'sub-minion')
+TMP_SYNDIC_MINION_CONF_DIR = os.path.join(TMP_CONF_DIR, 'syndic-minion')
+TMP_SYNDIC_MASTER_CONF_DIR = os.path.join(TMP_CONF_DIR, 'syndic-master')
+CONF_DIR = os.path.join(INTEGRATION_TEST_DIR, 'files', 'conf')
+PILLAR_DIR = os.path.join(FILES, 'pillar')
+TMP_SCRIPT_DIR = os.path.join(TMP, 'scripts')
+ENGINES_DIR = os.path.join(FILES, 'engines')
+LOG_HANDLERS_DIR = os.path.join(FILES, 'log_handlers')
+
+
+SCRIPT_TEMPLATES = {
+    'salt': [
+        'from salt.scripts import salt_main\n',
+        'if __name__ == \'__main__\':\n'
+        '    salt_main()'
+    ],
+    'salt-api': [
+        'import salt.cli\n',
+        'def main():\n',
+        '    sapi = salt.cli.SaltAPI()',
+        '    sapi.start()\n',
+        'if __name__ == \'__main__\':',
+        '    main()'
+    ],
+    'common': [
+        'from salt.scripts import salt_{0}\n',
+        'from salt.utils import is_windows\n\n',
+        'if __name__ == \'__main__\':\n',
+        '    if is_windows():\n',
+        '        import os.path\n',
+        '        import py_compile\n',
+        '        cfile = os.path.splitext(__file__)[0] + ".pyc"\n',
+        '        if not os.path.exists(cfile):\n',
+        '            py_compile.compile(__file__, cfile)\n',
+        '    salt_{0}()'
+    ]
+}
+
+
+class ScriptPathMixin(object):
+
+    def get_script_path(self, script_name):
+        '''
+        Return the path to a testing runtime script
+        '''
+        if not os.path.isdir(TMP_SCRIPT_DIR):
+            os.makedirs(TMP_SCRIPT_DIR)
+
+        script_path = os.path.join(TMP_SCRIPT_DIR,
+                                   'cli_{0}.py'.format(script_name.replace('-', '_')))
+
+        if not os.path.isfile(script_path):
+            log.info('Generating {0}'.format(script_path))
+
+            # Late import
+            import salt.utils
+
+            with salt.utils.fopen(script_path, 'w') as sfh:
+                script_template = SCRIPT_TEMPLATES.get(script_name, None)
+                if script_template is None:
+                    script_template = SCRIPT_TEMPLATES.get('common', None)
+                if script_template is None:
+                    raise RuntimeError(
+                        '{0} does not know how to handle the {1} script'.format(
+                            self.__class__.__name__,
+                            script_name
+                        )
+                    )
+                sfh.write(
+                    '#!{0}\n\n'.format(sys.executable) +
+                    'import sys\n' +
+                    'CODE_DIR="{0}"\n'.format(CODE_DIR) +
+                    'if CODE_DIR not in sys.path:\n' +
+                    '    sys.path.insert(0, CODE_DIR)\n\n' +
+                    '\n'.join(script_template).format(script_name.replace('salt-', ''))
+                )
+            fst = os.stat(script_path)
+            os.chmod(script_path, fst.st_mode | stat.S_IEXEC)
+
+        log.info('Returning script path %r', script_path)
+        return script_path

tests/support/processes.py

@@ -0,0 +1,192 @@
+# -*- coding: utf-8 -*-
+'''
+    :copyright: © 2017 by the SaltStack Team, see AUTHORS for more details.
+    :license: Apache 2.0, see LICENSE for more details.
+
+
+    tests.support.processes
+    ~~~~~~~~~~~~~~~~~~~~~~~
+
+    Process handling utilities
+'''
+
+# Import python libs
+from __future__ import absolute_import
+import logging
+
+# Import pytest-salt libs
+from pytestsalt.utils import SaltRunEventListener as PytestSaltRunEventListener
+from pytestsalt.utils import collect_child_processes, terminate_process  # pylint: disable=unused-import
+from pytestsalt.fixtures.daemons import Salt as PytestSalt
+from pytestsalt.fixtures.daemons import SaltKey as PytestSaltKey
+from pytestsalt.fixtures.daemons import SaltRun as PytestSaltRun
+from pytestsalt.fixtures.daemons import SaltCall as PytestSaltCall
+from pytestsalt.fixtures.daemons import SaltMaster as PytestSaltMaster
+from pytestsalt.fixtures.daemons import SaltMinion as PytestSaltMinion
+from pytestsalt.fixtures.daemons import SaltSyndic as PytestSaltSyndic
+
+# Import tests support libs
+from tests.support.paths import ScriptPathMixin
+
+log = logging.getLogger(__name__)
+
+
+class SaltRunEventListener(ScriptPathMixin, PytestSaltRunEventListener):
+    '''
+    Override this class's __init__ because there's no request argument since we're still
+    not running under pytest
+    '''
+
+
+class GetSaltRunFixtureMixin(ScriptPathMixin):
+    '''
+    Override this classes `get_salt_run_fixture` because we're still not running under pytest
+    '''
+
+    def get_salt_run_fixture(self):
+        pass
+
+    def get_salt_run_event_listener(self):
+        return SaltRunEventListener(None,
+                                    self.config,
+                                    self.config_dir,
+                                    self.bin_dir_path,
+                                    self.log_prefix,
+                                    cli_script_name='run')
+
+
+class Salt(ScriptPathMixin, PytestSalt):
+    '''
+    Class which runs salt-call commands
+    '''
+    def __init__(self, *args, **kwargs):
+        super(Salt, self).__init__(None, *args, **kwargs)
+
+
+class SaltCall(ScriptPathMixin, PytestSaltCall):
+    '''
+    Class which runs salt-call commands
+    '''
+    def __init__(self, *args, **kwargs):
+        super(SaltCall, self).__init__(None, *args, **kwargs)
+
+
+class SaltKey(ScriptPathMixin, PytestSaltKey):
+    '''
+    Class which runs salt-key commands
+    '''
+    def __init__(self, *args, **kwargs):
+        super(SaltKey, self).__init__(None, *args, **kwargs)
+
+
+class SaltRun(ScriptPathMixin, PytestSaltRun):
+    '''
+    Class which runs salt-run commands
+    '''
+    def __init__(self, *args, **kwargs):
+        super(SaltRun, self).__init__(None, *args, **kwargs)
+
+
+class SaltMinion(GetSaltRunFixtureMixin, PytestSaltMinion):
+    '''
+    Class which runs the salt-minion daemon
+    '''
+
+
+class SaltMaster(GetSaltRunFixtureMixin, PytestSaltMaster):
+    '''
+    Class which runs the salt-master daemon
+    '''
+
+
+class SaltSyndic(GetSaltRunFixtureMixin, PytestSaltSyndic):
+    '''
+    Class which runs the salt-syndic daemon
+    '''
+
+
+def start_daemon(daemon_name=None,
+                 daemon_id=None,
+                 daemon_log_prefix=None,
+                 daemon_cli_script_name=None,
+                 daemon_config=None,
+                 daemon_config_dir=None,
+                 daemon_class=None,
+                 bin_dir_path=None,
+                 fail_hard=False,
+                 start_timeout=10,
+                 slow_stop=False,
+                 environ=None,
+                 cwd=None):
+    '''
+    Returns a running salt daemon
+    '''
+    daemon_config['pytest_port'] = daemon_config['runtests_conn_check_port']
+    request = None
+    if fail_hard:
+        fail_method = RuntimeError
+    else:
+        fail_method = RuntimeWarning
+    log.info('[%s] Starting pytest %s(%s)', daemon_name, daemon_log_prefix, daemon_id)
+    attempts = 0
+    process = None
+    while attempts <= 3:  # pylint: disable=too-many-nested-blocks
+        attempts += 1
+        process = daemon_class(request,
+                               daemon_config,
+                               daemon_config_dir,
+                               bin_dir_path,
+                               daemon_log_prefix,
+                               cli_script_name=daemon_cli_script_name,
+                               slow_stop=slow_stop,
+                               environ=environ,
+                               cwd=cwd)
+        process.start()
+        if process.is_alive():
+            try:
+                connectable = process.wait_until_running(timeout=start_timeout)
+                if connectable is False:
+                    connectable = process.wait_until_running(timeout=start_timeout/2)
+                    if connectable is False:
+                        process.terminate()
+                        if attempts >= 3:
+                            fail_method(
+                                'The pytest {0}({1}) has failed to confirm running status '
+                                'after {2} attempts'.format(daemon_name, daemon_id, attempts))
+                        continue
+            except Exception as exc:  # pylint: disable=broad-except
+                log.exception('[%s] %s', daemon_log_prefix, exc, exc_info=True)
+                terminate_process(process.pid, kill_children=True, slow_stop=slow_stop)
+                if attempts >= 3:
+                    raise fail_method(str(exc))
+                continue
+            log.info(
+                '[%s] The pytest %s(%s) is running and accepting commands '
+                'after %d attempts',
+                daemon_log_prefix,
+                daemon_name,
+                daemon_id,
+                attempts
+            )
+
+            def stop_daemon():
+                log.info('[%s] Stopping pytest %s(%s)', daemon_log_prefix, daemon_name, daemon_id)
+                terminate_process(process.pid, kill_children=True, slow_stop=slow_stop)
+                log.info('[%s] pytest %s(%s) stopped', daemon_log_prefix, daemon_name, daemon_id)
+
+            # request.addfinalizer(stop_daemon)
+            return process
+        else:
+            terminate_process(process.pid, kill_children=True, slow_stop=slow_stop)
+            continue
+    else:   # pylint: disable=useless-else-on-loop
+            # Wrong, we have a return, its not useless
+        if process is not None:
+            terminate_process(process.pid, kill_children=True, slow_stop=slow_stop)
+        raise fail_method(
+            'The pytest {0}({1}) has failed to start after {2} attempts'.format(
+                daemon_name,
+                daemon_id,
+                attempts-1
+            )
+        )