More insight(context) on failure reports

This commit is contained in:
Pedro Algarvio 2019-04-11 11:10:04 +01:00
parent 93b6b95e6d
commit 7b173523d8
No known key found for this signature in database
GPG key ID: BB36BF6584A298FF
6 changed files with 298 additions and 127 deletions

View file

@ -5,6 +5,7 @@ Tests for existence of manpages
# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
import os
import pprint
import shutil
# Import Salt libs
@ -18,7 +19,6 @@ from tests.support.unit import skipIf
@skipIf(salt.utils.platform.is_windows(), 'minion is windows')
class ManTest(ModuleCase):
rootdir = os.path.join(TMP, 'mantest')
# Map filenames to search strings which should be in the manpage
manpages = {
'salt-cp.1': [
@ -82,15 +82,22 @@ class ManTest(ModuleCase):
}
def setUp(self):
if not self.run_function('mantest.install', [self.rootdir]):
self.fail('Failed to install salt to {0}'.format(self.rootdir))
@classmethod
def tearDownClass(cls):
try:
shutil.rmtree(cls.rootdir)
except OSError:
pass
self.rootdir = os.path.join(TMP, 'mantest')
self.addCleanup(shutil.rmtree, self.rootdir, ignore_errors=True)
if not os.path.exists(self.rootdir):
ret = self.run_function('mantest.install', [self.rootdir])
if not isinstance(ret, dict):
self.fail(
'The \'mantest.install\' command did not return the excepted dictionary. Output:\n{}'.format(
ret
)
)
if ret['retcode'] != 0:
self.fail(
'Failed to install. Full return dictionary:\n{}'.format(
pprint.pformat(ret)
)
)
def test_man(self):
'''

View file

@ -6,6 +6,7 @@ Helpers for testing man pages
from __future__ import absolute_import, print_function, unicode_literals
import os
import sys
import logging
# Import Salt libs
import salt.utils.files
@ -16,15 +17,20 @@ from salt.exceptions import CommandExecutionError
# Import Salt Tesing libs
from tests.support.paths import CODE_DIR
log = logging.getLogger(__name__)
def install(rootdir):
if not os.path.exists(rootdir):
os.makedirs(rootdir)
return __salt__['cmd.retcode'](
[sys.executable,
os.path.join(CODE_DIR, 'setup.py'),
'install', '--root={0}'.format(rootdir)]) == 0
return True
return __salt__['cmd.run_all'](
[
sys.executable,
os.path.join(CODE_DIR, 'setup.py'),
'install', '--root={0}'.format(rootdir)
],
redirect_stderr=True
)
def search(manpages, rootdir):

View file

@ -8,6 +8,7 @@ from __future__ import absolute_import, print_function, unicode_literals
import logging
import os
import time
import pprint
# Import Salt Testing libs
from tests.support.case import ModuleCase
@ -153,8 +154,10 @@ class GrainsAppendTestCase(ModuleCase):
GRAIN_KEY = 'salttesting-grain-key'
GRAIN_VAL = 'my-grain-val'
def tearDown(self):
def setUp(self):
# Start off with an empty list
self.run_function('grains.setval', [self.GRAIN_KEY, []])
self.addCleanup(self.run_function, 'grains.setval', [self.GRAIN_KEY, []])
def test_grains_append(self):
'''
@ -188,6 +191,8 @@ class GrainsAppendTestCase(ModuleCase):
'''
Tests the return of a grains.append call when val is passed in as a list.
'''
# Start off with an empty list, don't know if the flaky decorator runs the setUp function or not...
self.run_function('grains.setval', [self.GRAIN_KEY, []])
second_grain = self.GRAIN_VAL + '-2'
ret = self.run_function('grains.append', [self.GRAIN_KEY, [self.GRAIN_VAL, second_grain]])
self.assertEqual(ret[self.GRAIN_KEY], [self.GRAIN_VAL, second_grain])
@ -198,11 +203,11 @@ class GrainsAppendTestCase(ModuleCase):
but also ensure the grain is not listed twice.
'''
# First, add the test grain.
self.run_function('grains.append', [self.GRAIN_KEY, self.GRAIN_VAL])
append_1 = self.run_function('grains.append', [self.GRAIN_KEY, self.GRAIN_VAL])
# Call the function again, which results in a string message, as tested in
# test_grains_append_val_already_present above.
self.run_function('grains.append', [self.GRAIN_KEY, self.GRAIN_VAL])
append_2 = self.run_function('grains.append', [self.GRAIN_KEY, self.GRAIN_VAL])
# Now make sure the grain doesn't show up twice.
grains = self.run_function('grains.items')
@ -212,4 +217,13 @@ class GrainsAppendTestCase(ModuleCase):
count += 1
# We should only have hit the grain key once.
self.assertEqual(count, 1)
self.assertEqual(
count,
1,
msg='Count did not match({}!=1) while looking for key \'{}\'.\nFirst append return:\n{}\nSecond append return:\n{}'.format(
count,
self.GRAIN_KEY,
pprint.pformat(append_1),
pprint.pformat(append_2)
)
)

View file

@ -1,15 +1,14 @@
# -*- coding: utf-8 -*-
'''
:codeauthor: Pedro Algarvio (pedro@algarvio.me)
tests.integration.modules.pip
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
tests.integration.modules.pip
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
import os
import re
import pprint
import shutil
import tempfile
@ -139,16 +138,22 @@ class PipModuleTest(ModuleCase):
'pip.install', requirements=requirements_list,
bin_env=self.venv_dir, cwd=self.venv_dir
)
if not isinstance(ret, dict):
self.fail(
'The \'pip.install\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
try:
self.assertEqual(ret['retcode'], 0)
found = self.pip_successful_install(ret['stdout'])
self.assertTrue(found)
except (AssertionError, TypeError):
import pprint
pprint.pprint(ret)
raise
except KeyError as exc:
self.fail(
'The returned dictionary is missing an expected key. Error: \'{}\'. Dictionary: {}'.format(
exc,
pprint.pformat(ret)
)
)
@skip_if_not_root
def test_requirements_as_list_of_chains__cwd_not_set__absolute_file_path(self):
@ -175,17 +180,23 @@ class PipModuleTest(ModuleCase):
ret = self.run_function(
'pip.install', requirements=requirements_list, bin_env=self.venv_dir
)
if not isinstance(ret, dict):
self.fail(
'The \'pip.install\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
try:
self.assertEqual(ret['retcode'], 0)
found = self.pip_successful_install(ret['stdout'])
self.assertTrue(found)
except (AssertionError, TypeError):
import pprint
pprint.pprint(ret)
raise
except KeyError as exc:
self.fail(
'The returned dictionary is missing an expected key. Error: \'{}\'. Dictionary: {}'.format(
exc,
pprint.pformat(ret)
)
)
@skip_if_not_root
def test_requirements_as_list__absolute_file_path(self):
@ -205,16 +216,22 @@ class PipModuleTest(ModuleCase):
'pip.install', requirements=requirements_list, bin_env=self.venv_dir
)
found = self.pip_successful_install(ret['stdout'])
if not isinstance(ret, dict):
self.fail(
'The \'pip.install\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
try:
self.assertEqual(ret['retcode'], 0)
found = self.pip_successful_install(ret['stdout'])
self.assertTrue(found)
except (AssertionError, TypeError):
import pprint
pprint.pprint(ret)
raise
except KeyError as exc:
self.fail(
'The returned dictionary is missing an expected key. Error: \'{}\'. Dictionary: {}'.format(
exc,
pprint.pformat(ret)
)
)
@skip_if_not_root
def test_requirements_as_list__non_absolute_file_path(self):
@ -240,16 +257,23 @@ class PipModuleTest(ModuleCase):
'pip.install', requirements=requirements_list,
bin_env=self.venv_dir, cwd=req_cwd
)
if not isinstance(ret, dict):
self.fail(
'The \'pip.install\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
try:
self.assertEqual(ret['retcode'], 0)
found = self.pip_successful_install(ret['stdout'])
self.assertTrue(found)
except (AssertionError, TypeError):
import pprint
pprint.pprint(ret)
raise
except KeyError as exc:
self.fail(
'The returned dictionary is missing an expected key. Error: \'{}\'. Dictionary: {}'.format(
exc,
pprint.pformat(ret)
)
)
@skip_if_not_root
def test_chained_requirements__absolute_file_path(self):
@ -268,13 +292,21 @@ class PipModuleTest(ModuleCase):
ret = self.run_function(
'pip.install', requirements=req1_filename, bin_env=self.venv_dir
)
if not isinstance(ret, dict):
self.fail(
'The \'pip.install\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
try:
self.assertEqual(ret['retcode'], 0)
self.assertIn('installed pep8', ret['stdout'])
except (AssertionError, TypeError):
import pprint
pprint.pprint(ret)
raise
except KeyError as exc:
self.fail(
'The returned dictionary is missing an expected key. Error: \'{}\'. Dictionary: {}'.format(
exc,
pprint.pformat(ret)
)
)
@skip_if_not_root
def test_chained_requirements__non_absolute_file_path(self):
@ -298,13 +330,21 @@ class PipModuleTest(ModuleCase):
'pip.install', requirements=req1_filename, cwd=req_basepath,
bin_env=self.venv_dir
)
if not isinstance(ret, dict):
self.fail(
'The \'pip.install\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
try:
self.assertEqual(ret['retcode'], 0)
self.assertIn('installed pep8', ret['stdout'])
except (AssertionError, TypeError):
import pprint
pprint.pprint(ret)
raise
except KeyError as exc:
self.fail(
'The returned dictionary is missing an expected key. Error: \'{}\'. Dictionary: {}'.format(
exc,
pprint.pformat(ret)
)
)
@skip_if_not_root
def test_issue_4805_nested_requirements(self):
@ -320,34 +360,66 @@ class PipModuleTest(ModuleCase):
ret = self.run_function(
'pip.install', requirements=req1_filename, bin_env=self.venv_dir, timeout=300)
if self._check_download_error(ret['stdout']):
self.skipTest('Test skipped due to pip download error')
if not isinstance(ret, dict):
self.fail(
'The \'pip.install\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
try:
if self._check_download_error(ret['stdout']):
self.skipTest('Test skipped due to pip download error')
self.assertEqual(ret['retcode'], 0)
self.assertIn('installed pep8', ret['stdout'])
except (AssertionError, TypeError):
import pprint
pprint.pprint(ret)
raise
except KeyError as exc:
self.fail(
'The returned dictionary is missing an expected key. Error: \'{}\'. Dictionary: {}'.format(
exc,
pprint.pformat(ret)
)
)
def test_pip_uninstall(self):
# Let's create the testing virtualenv
self.run_function('virtualenv.create', [self.venv_dir])
ret = self.run_function('pip.install', ['pep8'], bin_env=self.venv_dir)
if self._check_download_error(ret['stdout']):
self.skipTest('Test skipped due to pip download error')
self.assertEqual(ret['retcode'], 0)
self.assertIn('installed pep8', ret['stdout'])
if not isinstance(ret, dict):
self.fail(
'The \'pip.install\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
try:
if self._check_download_error(ret['stdout']):
self.skipTest('Test skipped due to pip download error')
self.assertEqual(ret['retcode'], 0)
self.assertIn('installed pep8', ret['stdout'])
except KeyError as exc:
self.fail(
'The returned dictionary is missing an expected key. Error: \'{}\'. Dictionary: {}'.format(
exc,
pprint.pformat(ret)
)
)
ret = self.run_function(
'pip.uninstall', ['pep8'], bin_env=self.venv_dir
)
if not isinstance(ret, dict):
self.fail(
'The \'pip.uninstall\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
try:
self.assertEqual(ret['retcode'], 0)
self.assertIn('uninstalled pep8', ret['stdout'])
except AssertionError:
import pprint
pprint.pprint(ret)
raise
except KeyError as exc:
self.fail(
'The returned dictionary is missing an expected key. Error: \'{}\'. Dictionary: {}'.format(
exc,
pprint.pformat(ret)
)
)
def test_pip_install_upgrade(self):
# Create the testing virtualenv
@ -355,15 +427,24 @@ class PipModuleTest(ModuleCase):
ret = self.run_function(
'pip.install', ['pep8==1.3.4'], bin_env=self.venv_dir
)
if self._check_download_error(ret['stdout']):
self.skipTest('Test skipped due to pip download error')
if not isinstance(ret, dict):
self.fail(
'The \'pip.install\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
try:
if self._check_download_error(ret['stdout']):
self.skipTest('Test skipped due to pip download error')
self.assertEqual(ret['retcode'], 0)
self.assertIn('installed pep8', ret['stdout'])
except AssertionError:
import pprint
pprint.pprint(ret)
raise
except KeyError as exc:
self.fail(
'The returned dictionary is missing an expected key. Error: \'{}\'. Dictionary: {}'.format(
exc,
pprint.pformat(ret)
)
)
ret = self.run_function(
'pip.install',
@ -371,27 +452,44 @@ class PipModuleTest(ModuleCase):
bin_env=self.venv_dir,
upgrade=True
)
if self._check_download_error(ret['stdout']):
self.skipTest('Test skipped due to pip download error')
if not isinstance(ret, dict):
self.fail(
'The \'pip.install\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
try:
if self._check_download_error(ret['stdout']):
self.skipTest('Test skipped due to pip download error')
self.assertEqual(ret['retcode'], 0)
self.assertIn('installed pep8', ret['stdout'])
except AssertionError:
import pprint
pprint.pprint(ret)
raise
except KeyError as exc:
self.fail(
'The returned dictionary is missing an expected key. Error: \'{}\'. Dictionary: {}'.format(
exc,
pprint.pformat(ret)
)
)
ret = self.run_function(
'pip.uninstall', ['pep8'], bin_env=self.venv_dir
)
if not isinstance(ret, dict):
self.fail(
'The \'pip.uninstall\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
try:
self.assertEqual(ret['retcode'], 0)
self.assertIn('uninstalled pep8', ret['stdout'])
except AssertionError:
import pprint
pprint.pprint(ret)
raise
except KeyError as exc:
self.fail(
'The returned dictionary is missing an expected key. Error: \'{}\'. Dictionary: {}'.format(
exc,
pprint.pformat(ret)
)
)
def test_pip_install_multiple_editables(self):
editables = [
@ -406,17 +504,26 @@ class PipModuleTest(ModuleCase):
editable='{0}'.format(','.join(editables)),
bin_env=self.venv_dir
)
if self._check_download_error(ret['stdout']):
self.skipTest('Test skipped due to pip download error')
if not isinstance(ret, dict):
self.fail(
'The \'pip.install\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
try:
if self._check_download_error(ret['stdout']):
self.skipTest('Test skipped due to pip download error')
self.assertEqual(ret['retcode'], 0)
self.assertIn(
'Successfully installed Blinker SaltTesting', ret['stdout']
)
except AssertionError:
import pprint
pprint.pprint(ret)
raise
except KeyError as exc:
self.fail(
'The returned dictionary is missing an expected key. Error: \'{}\'. Dictionary: {}'.format(
exc,
pprint.pformat(ret)
)
)
def test_pip_install_multiple_editables_and_pkgs(self):
editables = [
@ -431,19 +538,28 @@ class PipModuleTest(ModuleCase):
editable='{0}'.format(','.join(editables)),
bin_env=self.venv_dir
)
if self._check_download_error(ret['stdout']):
self.skipTest('Test skipped due to pip download error')
if not isinstance(ret, dict):
self.fail(
'The \'pip.install\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
try:
if self._check_download_error(ret['stdout']):
self.skipTest('Test skipped due to pip download error')
self.assertEqual(ret['retcode'], 0)
for package in ('Blinker', 'SaltTesting', 'pep8'):
self.assertRegex(
ret['stdout'],
r'(?:.*)(Successfully installed)(?:.*)({0})(?:.*)'.format(package)
)
except AssertionError:
import pprint
pprint.pprint(ret)
raise
except KeyError as exc:
self.fail(
'The returned dictionary is missing an expected key. Error: \'{}\'. Dictionary: {}'.format(
exc,
pprint.pformat(ret)
)
)
@skipIf(not os.path.isfile('pip3'), 'test where pip3 is installed')
@skipIf(salt.utils.platform.is_windows(), 'test specific for linux usage of /bin/python')

View file

@ -3,6 +3,7 @@
# Import Python libs
from __future__ import absolute_import, print_function, unicode_literals
import os
import pprint
# Import Salt Testing libs
from tests.support.case import ModuleCase
@ -88,7 +89,19 @@ class PkgModuleTest(ModuleCase, SaltReturnAssertsMixin):
ret = self.run_function('pkg.mod_repo', [repo, 'comps=main'])
self.assertNotEqual(ret, {})
ret = self.run_function('pkg.get_repo', [repo])
self.assertEqual(ret['uri'], uri)
if not isinstance(ret, dict):
self.fail(
'The \'pkg.get_repo\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
self.assertEqual(
ret['uri'],
uri,
msg='The URI did not match. Full return:\n{}'.format(
pprint.pformat(ret)
)
)
elif os_grain == 'CentOS':
major_release = int(
self.run_function(

View file

@ -12,6 +12,7 @@ from __future__ import absolute_import, print_function, unicode_literals
import errno
import os
import glob
import pprint
import shutil
import sys
@ -56,7 +57,12 @@ class VirtualEnv(object):
def __enter__(self):
ret = self.test.run_function('virtualenv.create', [self.venv_dir])
self.test.assertEqual(ret['retcode'], 0)
self.test.assertEqual(
ret['retcode'], 0,
msg='Expected \'retcode\' key did not match. Full return dictionary:\n{}'.format(
pprint.pformat(ret)
)
)
def __exit__(self, exc_type, exc_value, traceback):
if os.path.isdir(self.venv_dir):
@ -216,7 +222,12 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin):
try:
# Let's create the testing virtualenv
ret = self.run_function('virtualenv.create', [venv_dir])
self.assertEqual(ret['retcode'], 0)
self.assertEqual(
ret['retcode'], 0,
msg='Expected \'retcode\' key did not match. Full return dictionary:\n{}'.format(
pprint.pformat(ret)
)
)
# Let's remove the pip binary
pip_bin = os.path.join(venv_dir, 'bin', 'pip')
@ -374,16 +385,19 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin):
ret = self.run_function('virtualenv.create', [venv_dir])
try:
try:
self.assertEqual(ret['retcode'], 0)
self.assertIn(
'New python executable',
ret['stdout']
self.assertEqual(
ret['retcode'], 0,
msg='Expected \'retcode\' key did not match. Full return dictionary:\n{}'.format(
pprint.pformat(ret)
)
except AssertionError:
import pprint
pprint.pprint(ret)
raise
)
self.assertIn(
'New python executable',
ret['stdout'],
msg='Expected STDOUT did not match. Full return dictionary:\n{}'.format(
pprint.pformat(ret)
)
)
# Let's install a fixed version pip over whatever pip was
# previously installed
@ -391,16 +405,17 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin):
'pip.install', ['pip==8.0'], upgrade=True,
bin_env=venv_dir
)
try:
self.assertEqual(ret['retcode'], 0)
self.assertIn(
'Successfully installed pip',
ret['stdout']
if not isinstance(ret, dict):
self.fail(
'The \'pip.install\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
except AssertionError:
import pprint
pprint.pprint(ret)
raise
self.assertEqual(ret['retcode'], 0)
self.assertIn(
'Successfully installed pip',
ret['stdout']
)
# Let's make sure we have pip 8.0 installed
self.assertEqual(
@ -413,14 +428,14 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin):
'pip.installed', name='pip==8.0.1', upgrade=True,
bin_env=venv_dir
)
try:
self.assertSaltTrueReturn(ret)
self.assertSaltStateChangesEqual(
ret, {'pip==8.0.1': 'Installed'})
except AssertionError:
import pprint
pprint.pprint(ret)
raise
if not isinstance(ret, dict):
self.fail(
'The \'pip.install\' command did not return the excepted dictionary. Output:\n{}'.format(ret)
)
self.assertSaltTrueReturn(ret)
self.assertSaltStateChangesEqual(ret, {'pip==8.0.1': 'Installed'})
finally:
if os.path.isdir(venv_dir):
shutil.rmtree(venv_dir, ignore_errors=True)