Replace flaky SPM man test

This replaces a test that has been intermittently failing, and also
expands it to check for _all_ man pages.

tests/integration/doc/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

tests/integration/doc/test_man.py

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+'''
+Tests for existence of manpages
+'''
+# Import python libs
+from __future__ import absolute_import, print_function, unicode_literals
+import os
+import shutil
+
+# Import Salt libs
+import salt.utils
+
+# Import Salt Testing libs
+from tests.support.case import ModuleCase
+from tests.support.paths import TMP
+from tests.support.unit import skipIf
+
+
+@skipIf(salt.utils.is_windows(), 'minion is windows')
+class ManTest(ModuleCase):
+    rootdir = os.path.join(TMP, 'mantest')
+    # Map filenames to search strings which should be in the manpage
+    manpages = {
+        'salt-cp.1': [
+            'salt-cp Documentation',
+            'copies files from the master',
+        ],
+        'salt-cloud.1': [
+            'Salt Cloud Command',
+            'Provision virtual machines in the cloud',
+        ],
+        'salt-call.1': [
+            'salt-call Documentation',
+            'run module functions locally',
+        ],
+        'salt-api.1': [
+            'salt-api Command',
+            'Start interfaces used to remotely connect',
+        ],
+        'salt-unity.1': [
+            'salt-unity Command',
+            'unified invocation wrapper',
+        ],
+        'salt-syndic.1': [
+            'salt-syndic Documentation',
+            'Salt syndic daemon',
+        ],
+        'salt-ssh.1': [
+            'salt-ssh Documentation',
+            'executed using only SSH',
+        ],
+        'salt-run.1': [
+            'salt-run Documentation',
+            'frontend command for executing',
+        ],
+        'salt-proxy.1': [
+            'salt-proxy Documentation',
+            'proxies these commands',
+        ],
+        'salt-minion.1': [
+            'salt-minion Documentation',
+            'Salt minion daemon',
+        ],
+        'salt-master.1': [
+            'salt-master Documentation',
+            'Salt master daemon',
+        ],
+        'salt-key.1': [
+            'salt-key Documentation',
+            'management of Salt server public keys',
+        ],
+        'salt.1': [
+            'allows for commands to be executed',
+        ],
+        'salt.7': [
+            'Salt Documentation',
+        ],
+        'spm.1': [
+            'Salt Package Manager Command',
+            'command for managing Salt packages',
+        ],
+    }
+
+    def setUp(self):
+        if not self.run_function('mantest.install', [self.rootdir]):
+            self.fail('Failed to install salt to {0}'.format(self.rootdir))
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            shutil.rmtree(cls.rootdir)
+        except OSError:
+            pass
+
+    def test_man(self):
+        '''
+        Make sure that man pages are installed
+        '''
+        ret = self.run_function('mantest.search', [self.manpages, self.rootdir])
+        # The above function returns True if successful and an exception (which
+        # will manifest in the return as a stringified exception) if
+        # unsuccessful. Therefore, a simple assertTrue is not sufficient.
+        if ret is not True:
+            self.fail(ret)

tests/integration/files/file/base/_modules/mantest.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+'''
+Helpers for testing man pages
+'''
+# Import python libs
+from __future__ import absolute_import, print_function, unicode_literals
+import os
+import sys
+
+# Import Salt libs
+import salt.utils
+from salt.exceptions import CommandExecutionError
+
+# Import Salt Tesing libs
+from tests.support.paths import CODE_DIR
+
+
+def install(rootdir):
+    if not os.path.exists(rootdir):
+        os.makedirs(rootdir)
+        return __salt__['cmd.retcode'](
+            [sys.executable,
+             os.path.join(CODE_DIR, 'setup.py'),
+             'install', '--root={0}'.format(rootdir)]) == 0
+    return True
+
+
+def search(manpages, rootdir):
+    manpage_fns = set(manpages)
+    manpage_paths = {}
+    for root, _, files in os.walk(rootdir):
+        if not manpage_fns:
+            # All manpages found, no need to keep walking
+            break
+        # Using list because we will be modifying the set during iteration
+        for manpage_fn in list(manpage_fns):
+            if manpage_fn in files:
+                manpage_path = salt.utils.path_join(root, manpage_fn)
+                manpage_paths[manpage_fn] = manpage_path
+                manpage_fns.remove(manpage_fn)
+
+    if manpage_fns:
+        raise CommandExecutionError(
+            'The following manpages were not found under {0}: {1}'.format(
+                rootdir,
+                ', '.join(sorted(manpage_fns))
+            )
+        )
+
+    failed = {}
+    for manpage in sorted(manpages):
+        with salt.utils.fopen(manpage_paths[manpage]) as fp_:
+            contents = salt.utils.to_unicode(fp_.read())
+        # Check for search string in contents
+        for search_string in manpages[manpage]:
+            if search_string not in contents:
+                failed.setdefault(manpage, []).append(
+                    'No match for search string \'{0}\' found in {1}'.format(
+                        search_string, manpage_paths[manpage]
+                    )
+                )
+        # Check for correct install dir
+        path = '/man{0}/'.format(manpage.rsplit('.', 1)[-1])
+        if path not in manpage_paths[manpage]:
+            failed.setdefault(manpage, []).append(
+                '{0} not found in manpage path {1}'.format(
+                    path, manpage_paths[manpage]
+                )
+            )
+
+    if failed:
+        raise CommandExecutionError('One or more manpages failed', info=failed)
+
+    return True

tests/integration/spm/test_man_spm.py

@@ -1,44 +0,0 @@
-# -*- coding: utf-8 -*-
-'''
-Tests man spm
-'''
-# Import python libs
-from __future__ import absolute_import
-import os
-import shutil
-import sys
-import tempfile
-
-# Import Salt Testing libs
-from tests.support.case import ModuleCase
-from tests.support.helpers import destructiveTest
-from tests.support.paths import CODE_DIR
-
-
-@destructiveTest
-class SPMManTest(ModuleCase):
-    '''
-    Validate man spm
-    '''
-
-    def setUp(self):
-        self.tmpdir = tempfile.mktemp()
-        os.mkdir(self.tmpdir)
-        self.run_function('cmd.run', ['{0} {1} install --root={2}'.format(
-            sys.executable,
-            os.path.join(CODE_DIR, 'setup.py'),
-            self.tmpdir
-        )])
-
-    def tearDown(self):
-        shutil.rmtree(self.tmpdir)
-
-    def test_man_spm(self):
-        '''
-        test man spm
-        '''
-        manpath = self.run_function('cmd.run', ['find {0} -name spm.1'.format(self.tmpdir)])
-        self.assertIn('/man1/', manpath)
-        cmd = self.run_function('cmd.run', ['man {0}'.format(manpath)])
-        self.assertIn('Salt Package Manager', cmd)
-        self.assertIn('command for managing Salt packages', cmd)

tests/runtests.py

@@ -112,6 +112,9 @@ TEST_SUITES = {
     'client':
        {'display_name': 'Client',
         'path': 'integration/client'},
+    'doc':
+       {'display_name': 'Documentation',
+        'path': 'integration/doc'},
     'ext_pillar':
        {'display_name': 'External Pillar',
         'path': 'integration/pillar'},
@@ -277,6 +280,15 @@ class SaltTestsuiteParser(SaltCoverageTestingParser):
             action='store_true',
             help='Run tests for client'
         )
+        self.test_selection_group.add_option(
+            '-d',
+            '--doc',
+            '--doc-tests',
+            dest='doc',
+            default=False,
+            action='store_true',
+            help='Run tests for documentation'
+        )
         self.test_selection_group.add_option(
             '-I',
             '--ext-pillar',