Fresh import on master

.gitignore

@@ -14,6 +14,7 @@ MANIFEST
 *.DS_Store
 .pytest_cache
 Pipfile.lock
+.mypy_cache/*
 
 # virtualenv
 # - ignores directories of a virtualenv when you create it right on

doc/topics/releases/neon.rst

@@ -4,6 +4,170 @@
 Salt Release Notes - Codename Neon
 ==================================
 
+Saltcheck Updates
+=================
+
+Available since 2018.3, the :py:func:`saltcheck module <salt.modules.saltcheck>`
+has been enhanced to:
+
+* Support saltenv environments
+* Associate tests with states by naming convention
+* Adds report_highstate_tests function
+* Adds empty and notempty assertions
+* Adds skip keyword
+* Adds print_result keyword
+* Adds assertion_section keyword
+* Use saltcheck.state_apply to run state.apply for test setup or teardown
+* Changes output to display test time
+* Works with salt-ssh
+
+Saltcheck provides unittest like functionality requiring only the knowledge of
+salt module execution and yaml. Saltcheck uses salt modules to return data, then
+runs an assertion against that return. This allows for testing with all the
+features included in salt modules.
+
+In order to run state and highstate saltcheck tests, a sub-folder in the state directory
+must be created and named ``saltcheck-tests``. Tests for a state should be created in files
+ending in ``*.tst`` and placed in the ``saltcheck-tests`` folder. ``tst`` files are run
+through the salt rendering system, enabling tests to be written in yaml (or renderer of choice),
+and include jinja, as well as the usual grain and pillar information. Like states, multiple tests can
+be specified in a ``tst`` file. Multiple ``tst`` files can be created in the ``saltcheck-tests``
+folder, and should be named the same as the associated state. The ``id`` of a test works in the
+same manner as in salt state files and should be unique and descriptive.
+
+Usage
+-----
+
+Example file system layout:
+
+.. code-block:: text
+
+    /srv/salt/apache/
+        init.sls
+        config.sls
+        saltcheck-tests/
+            init.tst
+            config.tst
+            deployment_validation.tst
+
+Tests can be run for each state by name, for all ``apache/saltcheck/*.tst`` files,
+or for all states assigned to the minion in top.sls. Tests may also be created
+with no associated state. These tests will be run through the use of
+``saltcheck.run_state_tests``, but will not be automatically run by
+``saltcheck.run_highstate_tests``.
+
+.. code-block:: bash
+
+    salt '*' saltcheck.run_state_tests apache,apache.config
+    salt '*' saltcheck.run_state_tests apache check_all=True
+    salt '*' saltcheck.run_highstate_tests
+    salt '*' saltcheck.run_state_tests apache.deployment_validation
+
+Example Tests
+-------------
+
+.. code-block:: jinja
+
+    {# will run the common salt state before further testing #}
+    setup_test_environment:
+      module_and_function: saltcheck.state_apply
+      args:
+        - common
+      pillar-data:
+        data: value
+
+    {% for package in ["apache2", "openssh"] %}
+    {# or another example #}
+    {# for package in salt['pillar.get']("packages") #}
+    jinja_test_{{ package }}_latest:
+      module_and_function: pkg.upgrade_available
+      args:
+        - {{ package }}
+      assertion: assertFalse
+    {% endfor %}
+
+    validate_user_present_and_shell:
+      module_and_function: user.info
+      args:
+        - root
+      assertion: assertEqual
+      expected-return: /bin/bash
+      assertion_section: shell
+      print_result: False
+
+    skip_test:
+      module_and_function: pkg.upgrade_available
+      args:
+        - apache2
+      assertion: assertFalse
+      skip: True
+
+Output Format Changes
+---------------------
+
+Saltcheck output has been enhanced to display the time taken per test. This results
+in a change to the output format.
+
+Previous Output:
+
+.. code-block:: text
+
+  local:
+    |_
+      ----------
+      ntp:
+          ----------
+          ntp-client-installed:
+              Pass
+          ntp-service-status:
+              Pass
+    |_
+      ----------
+      TEST RESULTS:
+          ----------
+          Failed:
+              0
+          Missing Tests:
+              0
+          Passed:
+              2
+
+New output:
+
+.. code-block:: text
+
+  local:
+    |_
+      ----------
+      ntp:
+          ----------
+          ntp-client-installed:
+              ----------
+              duration:
+                  1.0408
+              status:
+                  Pass
+          ntp-service-status:
+              ----------
+              duration:
+                  1.464
+              status:
+                  Pass
+    |_
+      ----------
+      TEST RESULTS:
+          ----------
+          Execution Time:
+              2.5048
+          Failed:
+              0
+          Missing Tests:
+              0
+          Passed:
+              2
+          Skipped:
+              0
+
 Slot Syntax Updates
 ===================
 
@@ -60,4 +224,4 @@ Returner Removal
 
 - The hipchat returner has been removed due to the service being retired. For users migrating
   to Slack, the :py:func:`slack <salt.returners.slack_returner>` returner may be a suitable
-  replacement.
+  replacement.

salt/client/ssh/wrapper/saltcheck.py

@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*-
+'''
+Wrap the saltcheck module to copy files to ssh minion before running tests
+'''
+# Import Python libs
+from __future__ import absolute_import, print_function
+import logging
+import tempfile
+import os
+import shutil
+import tarfile
+from contextlib import closing
+
+
+# Import salt libs
+import salt.utils.url
+import salt.utils.files
+import salt.utils.json
+
+log = logging.getLogger(__name__)
+
+
+def update_master_cache(states, saltenv='base'):
+    '''
+    Replace standard saltcheck version with similar logic but replacing cp.cache_dir with
+        generating files, tar'ing them up, copying tarball to remote host, extracting tar
+        to state cache directory, and cleanup of files
+    '''
+    cache = __opts__['cachedir']
+    state_cache = os.path.join(cache, 'files', saltenv)
+
+    # Setup for copying states to gendir
+    gendir = tempfile.mkdtemp()
+    trans_tar = salt.utils.files.mkstemp()
+    if 'cp.fileclient_{0}'.format(id(__opts__)) not in __context__:
+        __context__['cp.fileclient_{0}'.format(id(__opts__))] = \
+            salt.fileclient.get_file_client(__opts__)
+
+    # generate cp.list_states output and save to gendir
+    cp_output = salt.utils.json.dumps(__salt__['cp.list_states']())
+    cp_output_file = os.path.join(gendir, 'cp_output.txt')
+    with salt.utils.files.fopen(cp_output_file, 'w') as fp:
+        fp.write(cp_output)
+
+    # cp state directories to gendir
+    already_processed = []
+    sls_list = salt.utils.args.split_input(states)
+    for state_name in sls_list:
+        # generate low data for each state and save to gendir
+        state_low_file = os.path.join(gendir, state_name + '.low')
+        state_low_output = salt.utils.json.dumps(__salt__['state.show_low_sls'](state_name))
+        with salt.utils.files.fopen(state_low_file, 'w') as fp:
+            fp.write(state_low_output)
+
+        state_name = state_name.replace(".", os.sep)
+        if state_name in already_processed:
+            log.debug("Already cached state for %s", state_name)
+        else:
+            file_copy_file = os.path.join(gendir, state_name + '.copy')
+            log.debug('copying %s to %s', state_name, gendir)
+            qualified_name = salt.utils.url.create(state_name, saltenv)
+            # Duplicate cp.get_dir to gendir
+            copy_result = __context__['cp.fileclient_{0}'.format(id(__opts__))].get_dir(
+                qualified_name, gendir, saltenv)
+            if copy_result:
+                copy_result = [dir.replace(gendir, state_cache) for dir in copy_result]
+                copy_result_output = salt.utils.json.dumps(copy_result)
+                with salt.utils.files.fopen(file_copy_file, 'w') as fp:
+                    fp.write(copy_result_output)
+                already_processed.append(state_name)
+            else:
+                # If files were not copied, assume state.file.sls was given and just copy state
+                state_name = os.path.dirname(state_name)
+                file_copy_file = os.path.join(gendir, state_name + '.copy')
+                if state_name in already_processed:
+                    log.debug('Already cached state for %s', state_name)
+                else:
+                    qualified_name = salt.utils.url.create(state_name, saltenv)
+                    copy_result = __context__['cp.fileclient_{0}'.format(id(__opts__))].get_dir(
+                        qualified_name, gendir, saltenv)
+                    if copy_result:
+                        copy_result = [dir.replace(gendir, state_cache) for dir in copy_result]
+                        copy_result_output = salt.utils.json.dumps(copy_result)
+                        with salt.utils.files.fopen(file_copy_file, 'w') as fp:
+                            fp.write(copy_result_output)
+                        already_processed.append(state_name)
+
+    # turn gendir into tarball and remove gendir
+    try:
+        # cwd may not exist if it was removed but salt was run from it
+        cwd = os.getcwd()
+    except OSError:
+        cwd = None
+    os.chdir(gendir)
+    with closing(tarfile.open(trans_tar, 'w:gz')) as tfp:
+        for root, dirs, files in salt.utils.path.os_walk(gendir):
+            for name in files:
+                full = os.path.join(root, name)
+                tfp.add(full[len(gendir):].lstrip(os.sep))
+    if cwd:
+        os.chdir(cwd)
+    shutil.rmtree(gendir)
+
+    # Copy tarfile to ssh host
+    single = salt.client.ssh.Single(
+            __opts__,
+            '',
+            **__salt__.kwargs)
+    thin_dir = __opts__['thin_dir']
+    ret = single.shell.send(trans_tar, thin_dir)
+
+    # Clean up local tar
+    try:
+        os.remove(trans_tar)
+    except (OSError, IOError):
+        pass
+
+    tar_path = os.path.join(thin_dir, os.path.basename(trans_tar))
+    # Extract remote tarball to cache directory and remove tar file
+    # TODO this could be better handled by a single state/connection due to ssh overhead
+    ret = __salt__['file.mkdir'](state_cache)
+    ret = __salt__['archive.tar']('xf', tar_path, dest=state_cache)
+    ret = __salt__['file.remove'](tar_path)
+
+    return ret
+
+
+def run_state_tests(states, saltenv='base', check_all=False):
+    '''
+    Define common functions to activite this wrapping module and tar copy.
+    After file copies are finished, run the usual local saltcheck function
+    '''
+    ret = update_master_cache(states, saltenv)
+    ret = __salt__['saltcheck.run_state_tests_ssh'](states, saltenv=saltenv, check_all=check_all)
+    return ret
+
+
+def run_highstate_tests(saltenv='base'):
+    '''
+    Lookup top files for minion, pass results to wrapped run_state_tests for copy and run
+    '''
+    top_states = __salt__['state.show_top']().get(saltenv)
+    state_string = ','.join(top_states)
+    ret = run_state_tests(state_string, saltenv)
+    return ret

tests/integration/files/file/base/validate-saltcheck/init.sls

@@ -0,0 +1,3 @@
+saltcheck-test-pass:
+  test.succeed_without_changes:
+    - name: testing-saltcheck

tests/integration/files/file/base/validate-saltcheck/saltcheck-tests/check_all.tst

@@ -0,0 +1,7 @@
+check_all_validate:
+  module_and_function: test.echo
+  args:
+    - "check"
+  kwargs:
+  assertion: assertEqual
+  expected_return:  'check'

tests/integration/files/file/base/validate-saltcheck/saltcheck-tests/init.tst

@@ -0,0 +1,7 @@
+echo_test_hello:
+  module_and_function: test.echo
+  args:
+    - "hello"
+  kwargs:
+  assertion: assertEqual
+  expected_return:  'hello'

tests/integration/files/file/prod/validate-saltcheck/init.sls

@@ -0,0 +1,3 @@
+saltcheck-prod-test-pass:
+  test.succeed_without_changes:
+    - name: testing-saltcheck-prodenv

tests/integration/files/file/prod/validate-saltcheck/saltcheck-tests/check_all.tst

@@ -0,0 +1,7 @@
+check_all_validate_prod:
+  module_and_function: test.echo
+  args:
+    - "check-prod"
+  kwargs:
+  assertion: assertEqual
+  expected_return: 'check-prod'

tests/integration/files/file/prod/validate-saltcheck/saltcheck-tests/init.tst

@@ -0,0 +1,7 @@
+echo_test_prod_env:
+  module_and_function: test.echo
+  args:
+    - "test-prod"
+  kwargs:
+  assertion: assertEqual
+  expected_return:  'test-prod'

tests/integration/modules/test_saltcheck.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+'''
+Test the saltcheck module
+'''
+# Import python libs
+from __future__ import absolute_import, print_function, unicode_literals
+
+# Import Salt Testing libs
+from tests.support.case import ModuleCase
+
+
+class SaltcheckModuleTest(ModuleCase):
+    '''
+    Test the saltcheck module
+    '''
+    def test_saltcheck_run(self):
+        '''
+        saltcheck.run_test
+        '''
+        saltcheck_test = {"module_and_function": "test.echo",
+                        "assertion": "assertEqual",
+                        "expected_return": "This works!",
+                        "args": ["This works!"]}
+        ret = self.run_function('saltcheck.run_test', test=saltcheck_test)
+        self.assertDictContainsSubset({'status': 'Pass'}, ret)
+
+    def test_saltcheck_state(self):
+        '''
+        saltcheck.run_state_tests
+        '''
+        saltcheck_test = 'validate-saltcheck'
+        ret = self.run_function('saltcheck.run_state_tests', [saltcheck_test])
+        self.assertDictContainsSubset({'status': 'Pass'}, ret[0]['validate-saltcheck']['echo_test_hello'])
+
+    def test_topfile_validation(self):
+        '''
+        saltcheck.run_highstate_tests
+        '''
+        expected_top_states = self.run_function('state.show_top').get('base', [])
+        expected_top_states.append('TEST RESULTS')
+        ret = self.run_function('saltcheck.run_highstate_tests')
+        for top_state_dict in ret:
+            self.assertIn(list(top_state_dict)[0], expected_top_states)
+
+    def test_saltcheck_checkall(self):
+        '''
+        Validate saltcheck.run_state_tests check_all for the default saltenv of base.
+        validate-saltcheck state hosts a saltcheck-tests directory with 2 .tst files. By running
+          check_all=True, both files should be found and show passed results.
+        '''
+        saltcheck_test = 'validate-saltcheck'
+        ret = self.run_function('saltcheck.run_state_tests', [saltcheck_test], check_all=True)
+        self.assertDictContainsSubset({'status': 'Pass'}, ret[0]['validate-saltcheck']['echo_test_hello'])
+        self.assertDictContainsSubset({'status': 'Pass'}, ret[0]['validate-saltcheck']['check_all_validate'])
+
+    def test_saltcheck_checkall_saltenv(self):
+        '''
+        Validate saltcheck.run_state_tests check_all for the prod saltenv
+        validate-saltcheck state hosts a saltcheck-tests directory with 2 .tst files. By running
+          check_all=True, both files should be found and show passed results.
+        '''
+        saltcheck_test = 'validate-saltcheck'
+        ret = self.run_function('saltcheck.run_state_tests', [saltcheck_test], saltenv='prod', check_all=True)
+        self.assertDictContainsSubset({'status': 'Pass'}, ret[0]['validate-saltcheck']['echo_test_prod_env'])
+        self.assertDictContainsSubset({'status': 'Pass'}, ret[0]['validate-saltcheck']['check_all_validate_prod'])

tests/integration/ssh/test_saltcheck.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+
+# Import Python libs
+from __future__ import absolute_import, print_function, unicode_literals
+
+# Import Salt Testing Libs
+from tests.support.case import SSHCase
+from tests.support.unit import skipIf
+
+# Import Salt Libs
+import salt.utils.platform
+
+
+@skipIf(salt.utils.platform.is_windows(), 'salt-ssh not available on Windows')
+class SSHSaltcheckTest(SSHCase):
+    '''
+    testing saltcheck with salt-ssh
+    '''
+    def test_saltcheck_run_test(self):
+        '''
+        test saltcheck.run_test with salt-ssh
+        '''
+        saltcheck_test = {"module_and_function": "test.echo",
+                        "assertion": "assertEqual",
+                        "expected-return": "Test Works",
+                        "args": ["Test Works"]}
+        ret = self.run_function('saltcheck.run_test', test=saltcheck_test)
+        self.assertDictContainsSubset({'status': 'Pass'}, ret)
+
+    def test_saltcheck_state(self):
+        '''
+        saltcheck.run_state_tests
+        '''
+        saltcheck_test = 'validate-saltcheck'
+        ret = self.run_function('saltcheck.run_state_tests', [saltcheck_test])
+        self.assertDictContainsSubset({'status': 'Pass'}, ret[0]['validate-saltcheck']['echo_test_hello'])

tests/support/case.py

@@ -135,7 +135,7 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
         return self.run_script('salt', arg_str, with_retcode=with_retcode, catch_stderr=catch_stderr, timeout=timeout)
 
     def run_ssh(self, arg_str, with_retcode=False, timeout=25,
-                catch_stderr=False, wipe=False, raw=False):
+                catch_stderr=False, wipe=False, raw=False, **kwargs):
         '''
         Execute salt-ssh
         '''
@@ -147,7 +147,12 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
             os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'roster'),
             arg_str
         )
-        return self.run_script('salt-ssh', arg_str, with_retcode=with_retcode, catch_stderr=catch_stderr, raw=True, timeout=timeout)
+        return self.run_script('salt-ssh',
+                               arg_str, with_retcode=with_retcode,
+                               catch_stderr=catch_stderr,
+                               raw=True,
+                               timeout=timeout,
+                               **kwargs)
 
     def run_run(self,
                 arg_str,
@@ -260,7 +265,8 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
                    timeout=15,
                    raw=False,
                    popen_kwargs=None,
-                   log_output=None):
+                   log_output=None,
+                   **kwargs):
         '''
         Execute a script with the given argument string
 
@@ -300,6 +306,11 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin):
             cmd += 'python{0}.{1} '.format(*sys.version_info)
         cmd += '{0} '.format(script_path)
         cmd += '{0} '.format(arg_str)
+        if kwargs:
+            # late import
+            import salt.utils.json
+            for key, value in kwargs.items():
+                cmd += "'{0}={1} '".format(key, salt.utils.json.dumps(value))
 
         tmp_file = tempfile.SpooledTemporaryFile()
 
@@ -542,7 +553,7 @@ class ShellCase(ShellTestCase, AdaptedConfigurationTestCaseMixin, ScriptPathMixi
         return ret
 
     def run_ssh(self, arg_str, with_retcode=False, catch_stderr=False,  # pylint: disable=W0221
-                timeout=RUN_TIMEOUT, wipe=True, raw=False):
+                timeout=RUN_TIMEOUT, wipe=True, raw=False, **kwargs):
         '''
         Execute salt-ssh
         '''
@@ -558,8 +569,9 @@ class ShellCase(ShellTestCase, AdaptedConfigurationTestCaseMixin, ScriptPathMixi
                               with_retcode=with_retcode,
                               catch_stderr=catch_stderr,
                               timeout=timeout,
-                              raw=True)
-        log.debug('Result of run_ssh for command \'%s\': %s', arg_str, ret)
+                              raw=True,
+                              **kwargs)
+        log.debug('Result of run_ssh for command \'%s %s\': %s', arg_str, kwargs, ret)
         return ret
 
     def run_run(self, arg_str, with_retcode=False, catch_stderr=False,
@@ -1018,8 +1030,8 @@ class SSHCase(ShellCase):
         We use a 180s timeout here, which some slower systems do end up needing
         '''
         ret = self.run_ssh(self._arg_str(function, arg), timeout=timeout,
-                           wipe=wipe, raw=raw)
-        log.debug('SSHCase run_function executed %s with arg %s', function, arg)
+                           wipe=wipe, raw=raw, **kwargs)
+        log.debug('SSHCase run_function executed %s with arg %s and kwargs %s', function, arg, kwargs)
         log.debug('SSHCase JSON return: %s', ret)
 
         # Late import

tests/unit/modules/test_saltcheck.py

@@ -27,7 +27,7 @@ except:
 
 
 @skipIf(NO_MOCK, NO_MOCK_REASON)
-class LinuxSysctlTestCase(TestCase, LoaderModuleMockMixin):
+class SaltcheckTestCase(TestCase, LoaderModuleMockMixin):
     '''
     TestCase for salt.modules.saltcheck module
     '''
@@ -37,11 +37,12 @@ class LinuxSysctlTestCase(TestCase, LoaderModuleMockMixin):
         local_opts = salt.config.minion_config(
             os.path.join(syspaths.CONFIG_DIR, 'minion'))
         local_opts['file_client'] = 'local'
+        local_opts['conf_file'] = '/etc/salt/minion'
         patcher = patch('salt.config.minion_config',
                         MagicMock(return_value=local_opts))
         patcher.start()
         self.addCleanup(patcher.stop)
-        return {saltcheck: {}}
+        return {saltcheck: {'__opts__': local_opts}}
 
     def test_call_salt_command(self):
         '''test simple test.echo module'''
@@ -50,13 +51,9 @@ class LinuxSysctlTestCase(TestCase, LoaderModuleMockMixin):
                                              'cp.cache_master': MagicMock(return_value=[True])
                                              }):
             sc_instance = saltcheck.SaltCheck()
-            returned = sc_instance.call_salt_command(fun="test.echo", args=['hello'], kwargs=None)
+            returned = sc_instance._call_salt_command(fun="test.echo", args=['hello'], kwargs=None)
             self.assertEqual(returned, 'hello')
 
-    def test_update_master_cache(self):
-        '''test master cache'''
-        self.assertTrue(saltcheck.update_master_cache)
-
     def test_call_salt_command2(self):
         '''test simple test.echo module again'''
         with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True),
@@ -64,7 +61,7 @@ class LinuxSysctlTestCase(TestCase, LoaderModuleMockMixin):
                                              'cp.cache_master': MagicMock(return_value=[True])
                                              }):
             sc_instance = saltcheck.SaltCheck()
-            returned = sc_instance.call_salt_command(fun="test.echo", args=['hello'], kwargs=None)
+            returned = sc_instance._call_salt_command(fun="test.echo", args=['hello'], kwargs=None)
             self.assertNotEqual(returned, 'not-hello')
 
     def test__assert_equal1(self):
@@ -321,6 +318,42 @@ class LinuxSysctlTestCase(TestCase, LoaderModuleMockMixin):
             mybool = sc_instance._SaltCheck__assert_less_equal(aaa, bbb)
             self.assertEqual(mybool, 'Pass')
 
+    def test__assert_empty(self):
+        '''test'''
+        with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True),
+                                             'cp.cache_master': MagicMock(return_value=[True])
+                                             }):
+            sc_instance = saltcheck.SaltCheck()
+            mybool = sc_instance._SaltCheck__assert_empty("")
+            self.assertEqual(mybool, 'Pass')
+
+    def test__assert_empty_fail(self):
+        '''test'''
+        with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True),
+                                             'cp.cache_master': MagicMock(return_value=[True])
+                                             }):
+            sc_instance = saltcheck.SaltCheck()
+            mybool = sc_instance._SaltCheck__assert_empty("data")
+            self.assertNotEqual(mybool, 'Pass')
+
+    def test__assert__not_empty(self):
+        '''test'''
+        with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True),
+                                             'cp.cache_master': MagicMock(return_value=[True])
+                                             }):
+            sc_instance = saltcheck.SaltCheck()
+            mybool = sc_instance._SaltCheck__assert_not_empty("data")
+            self.assertEqual(mybool, 'Pass')
+
+    def test__assert__not_empty_fail(self):
+        '''test'''
+        with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True),
+                                             'cp.cache_master': MagicMock(return_value=[True])
+                                             }):
+            sc_instance = saltcheck.SaltCheck()
+            mybool = sc_instance._SaltCheck__assert_not_empty("")
+            self.assertNotEqual(mybool, 'Pass')
+
     def test_run_test_1(self):
         '''test'''
         with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True),
@@ -329,7 +362,252 @@ class LinuxSysctlTestCase(TestCase, LoaderModuleMockMixin):
                                              'cp.cache_master': MagicMock(return_value=[True])}):
             returned = saltcheck.run_test(test={"module_and_function": "test.echo",
                                                 "assertion": "assertEqual",
-                                                "expected-return": "This works!",
+                                                "expected_return": "This works!",
                                                 "args": ["This works!"]
                                                 })
-            self.assertEqual(returned, 'Pass')
+            self.assertEqual(returned['status'], 'Pass')
+
+    def test_report_highstate_tests(self):
+        '''test report_highstate_tests'''
+        expected_output = {'TEST REPORT RESULTS': {
+                           'States missing tests': ['state1'],
+                           'Missing Tests': 1,
+                           'States with tests': ['found']
+                          }}
+        with patch('salt.modules.saltcheck._get_top_states') as mocked_get_top:
+            mocked_get_top.return_value = ['state1', 'found']
+            with patch('salt.modules.saltcheck.StateTestLoader') as mocked_stl:
+                instance = mocked_stl.return_value
+                instance.found_states = ['found']
+                returned = saltcheck.report_highstate_tests()
+                self.assertEqual(returned, expected_output)
+
+    def test_validation(self):
+        '''test validation of tests'''
+        sc_instance = saltcheck.SaltCheck()
+
+        # Fail on empty test
+        test_dict = {}
+        expected_return = False
+        val_ret = sc_instance._SaltCheck__is_valid_test(test_dict)
+        self.assertEqual(val_ret, expected_return)
+
+        # Succeed on standard test
+        test_dict = {
+                    'module_and_function': 'test.echo',
+                    'args': ["hello"],
+                    'kwargs': {},
+                    'assertion': 'assertEqual',
+                    'expected_return':  'hello'
+                    }
+        expected_return = True
+        with patch.dict(saltcheck.__salt__, {'sys.list_modules': MagicMock(return_value=['test']),
+                                             'sys.list_functions': MagicMock(return_value=['test.echo'])
+                                            }):
+            val_ret = sc_instance._SaltCheck__is_valid_test(test_dict)
+            self.assertEqual(val_ret, expected_return)
+
+        # Succeed on standard test with older expected-return syntax
+        test_dict = {
+                    'module_and_function': 'test.echo',
+                    'args': ["hello"],
+                    'kwargs': {},
+                    'assertion': 'assertEqual',
+                    'expected-return':  'hello'
+                    }
+        expected_return = True
+        with patch.dict(saltcheck.__salt__, {'sys.list_modules': MagicMock(return_value=['test']),
+                                             'sys.list_functions': MagicMock(return_value=['test.echo'])
+                                            }):
+            val_ret = sc_instance._SaltCheck__is_valid_test(test_dict)
+            self.assertEqual(val_ret, expected_return)
+
+        # Do not require expected_return for some assertions
+        assertions = ["assertEmpty",
+                      "assertNotEmpty",
+                      "assertTrue",
+                      "assertFalse"]
+        for assertion in assertions:
+            test_dict = {
+                        'module_and_function': 'test.echo',
+                        'args': ["hello"]
+                        }
+            test_dict['assertion'] = assertion
+            expected_return = True
+            with patch.dict(saltcheck.__salt__, {'sys.list_modules': MagicMock(return_value=['test']),
+                                                'sys.list_functions': MagicMock(return_value=['test.echo'])
+                                                }):
+                val_ret = sc_instance._SaltCheck__is_valid_test(test_dict)
+                self.assertEqual(val_ret, expected_return)
+
+        # Fail on invalid module
+        test_dict = {
+                    'module_and_function': 'broken.echo',
+                    'args': ["hello"],
+                    'kwargs': {},
+                    'assertion': 'assertEqual',
+                    'expected_return':  'hello'
+                    }
+        expected_return = False
+        with patch.dict(saltcheck.__salt__, {'sys.list_modules': MagicMock(return_value=['test']),
+                                             'sys.list_functions': MagicMock(return_value=['test.echo'])
+                                            }):
+            val_ret = sc_instance._SaltCheck__is_valid_test(test_dict)
+            self.assertEqual(val_ret, expected_return)
+
+        # Fail on invalid function
+        test_dict = {
+                    'module_and_function': 'test.broken',
+                    'args': ["hello"],
+                    'kwargs': {},
+                    'assertion': 'assertEqual',
+                    'expected_return':  'hello'
+                    }
+        expected_return = False
+        with patch.dict(saltcheck.__salt__, {'sys.list_modules': MagicMock(return_value=['test']),
+                                             'sys.list_functions': MagicMock(return_value=['test.echo'])
+                                            }):
+            val_ret = sc_instance._SaltCheck__is_valid_test(test_dict)
+            self.assertEqual(val_ret, expected_return)
+
+        # Fail on missing expected_return
+        test_dict = {
+                    'module_and_function': 'test.echo',
+                    'args': ["hello"],
+                    'kwargs': {},
+                    'assertion': 'assertEqual'
+                    }
+        expected_return = False
+        with patch.dict(saltcheck.__salt__, {'sys.list_modules': MagicMock(return_value=['test']),
+                                             'sys.list_functions': MagicMock(return_value=['test.echo'])
+                                            }):
+            val_ret = sc_instance._SaltCheck__is_valid_test(test_dict)
+            self.assertEqual(val_ret, expected_return)
+
+        # Fail on empty expected_return
+        test_dict = {
+                    'module_and_function': 'test.echo',
+                    'args': ["hello"],
+                    'kwargs': {},
+                    'assertion': 'assertEqual',
+                    'expected_return':  None
+                    }
+        expected_return = False
+        with patch.dict(saltcheck.__salt__, {'sys.list_modules': MagicMock(return_value=['test']),
+                                             'sys.list_functions': MagicMock(return_value=['test.echo'])
+                                            }):
+            val_ret = sc_instance._SaltCheck__is_valid_test(test_dict)
+            self.assertEqual(val_ret, expected_return)
+
+        # Succeed on m_and_f saltcheck.state_apply with only args
+        test_dict = {
+                    'module_and_function': 'saltcheck.state_apply',
+                    'args': ["common"]
+                    }
+        expected_return = True
+        with patch.dict(saltcheck.__salt__, {'sys.list_modules': MagicMock(return_value=['saltcheck']),
+                                             'sys.list_functions': MagicMock(return_value=['saltcheck.state_apply'])
+                                            }):
+            val_ret = sc_instance._SaltCheck__is_valid_test(test_dict)
+            self.assertEqual(val_ret, expected_return)
+
+    def test_sls_path_generation(self):
+        '''test generation of sls paths'''
+        with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value='saltcheck-tests')}):
+            testLoader = saltcheck.StateTestLoader()
+
+            state_name = 'teststate'
+            expected_return = ['salt://teststate/saltcheck-tests',
+                               'salt:///saltcheck-tests']
+            ret = testLoader._generate_sls_path(state_name)
+            self.assertEqual(ret, expected_return)
+
+            state_name = 'teststate.long.path'
+            expected_return = ['salt://teststate/long/path/saltcheck-tests',
+                               'salt://teststate/long/saltcheck-tests',
+                               'salt://teststate/saltcheck-tests']
+            ret = testLoader._generate_sls_path(state_name)
+            self.assertEqual(ret, expected_return)
+
+            state_name = 'teststate.really.long.path'
+            expected_return = ['salt://teststate/really/long/path/saltcheck-tests',
+                               'salt://teststate/really/long/saltcheck-tests',
+                               'salt://teststate/saltcheck-tests']
+            ret = testLoader._generate_sls_path(state_name)
+            self.assertEqual(ret, expected_return)
+
+    def test_generate_output(self):
+        # passing states
+        sc_results = {'a_state':
+                        {'test_id1': {'status': 'Pass', 'duration': 1.987},
+                        'test_id2': {'status': 'Pass', 'duration': 1.123}}}
+        expected_output = [{'a_state':
+                            {'test_id1': {'status': 'Pass', 'duration': 1.987},
+                            'test_id2': {'status': 'Pass', 'duration': 1.123}}},
+                            {'TEST RESULTS':
+                                {'Execution Time': 3.11,
+                                'Passed': 2,
+                                'Failed': 0,
+                                'Skipped': 0,
+                                'Missing Tests': 0
+                            }
+                           }]
+        ret = saltcheck._generate_out_list(sc_results)
+        self.assertEqual(ret, expected_output)
+
+        # Skipped
+        sc_results = {'a_state':
+                        {'test_id1': {'status': 'Skip', 'duration': 1.987},
+                        'test_id2': {'status': 'Pass', 'duration': 1.123}}}
+        expected_output = [{'a_state':
+                            {'test_id1': {'status': 'Skip', 'duration': 1.987},
+                            'test_id2': {'status': 'Pass', 'duration': 1.123}}},
+                            {'TEST RESULTS':
+                                {'Execution Time': 3.11,
+                                'Passed': 1,
+                                'Failed': 0,
+                                'Skipped': 1,
+                                'Missing Tests': 0
+                            }
+                           }]
+        ret = saltcheck._generate_out_list(sc_results)
+        self.assertEqual(ret, expected_output)
+
+        # Failed (does not test setting __context__)
+        sc_results = {'a_state':
+                        {'test_id1': {'status': 'Failed', 'duration': 1.987},
+                        'test_id2': {'status': 'Pass', 'duration': 1.123}}}
+        expected_output = [{'a_state':
+                            {'test_id1': {'status': 'Failed', 'duration': 1.987},
+                            'test_id2': {'status': 'Pass', 'duration': 1.123}}},
+                            {'TEST RESULTS':
+                                {'Execution Time': 3.11,
+                                'Passed': 1,
+                                'Failed': 1,
+                                'Skipped': 0,
+                                'Missing Tests': 0
+                            }
+                           }]
+        ret = saltcheck._generate_out_list(sc_results)
+        self.assertEqual(ret, expected_output)
+
+        # missing states
+        sc_results = {'a_state':
+                        {'test_id1': {'status': 'Pass', 'duration': 1.987},
+                        'test_id2': {'status': 'Pass', 'duration': 1.123}},
+                      'b_state': {}
+                     }
+        expected_output = [{'a_state':
+                            {'test_id1': {'status': 'Pass', 'duration': 1.987},
+                            'test_id2': {'status': 'Pass', 'duration': 1.123}}},
+                            {'b_state': {}},
+                            {'TEST RESULTS':
+                                {'Execution Time': 3.11,
+                                'Passed': 2,
+                                'Failed': 0,
+                                'Skipped': 0,
+                                'Missing Tests': 1
+                            }
+                           }]
+        ret = saltcheck._generate_out_list(sc_results)
+        self.assertEqual(ret, expected_output)

tests/unit/test_module_names.py

@@ -187,6 +187,7 @@ class BadTestModuleNamesTestCase(TestCase):
             'integration.ssh.test_mine',
             'integration.ssh.test_pillar',
             'integration.ssh.test_raw',
+            'integration.ssh.test_saltcheck',
             'integration.ssh.test_state',
             'integration.states.test_compiler',
             'integration.states.test_handle_error',