saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #48943 from rallytime/flaky-tests

Browse source 
Mark some shell and runner integration tests as flaky
This commit is contained in:
Nicole Thomas 2018-08-06 09:08:13 -04:00 • committed by GitHub
commit 928d688d65
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 31 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
tests/integration/runners/test_state.py
View file

@ -19,8 +19,9 @@ from salt.ext.six.moves import queue
# Import Salt Testing Libs
from tests.support.case import ShellCase
from tests.support.unit import skipIf
from tests.support.helpers import flaky
from tests.support.paths import TMP
from tests.support.unit import skipIf
# Import Salt Libs
import salt.utils
@ -53,7 +54,6 @@ class StateRunnerTest(ShellCase):
Also test against some sample "good" output that would be included in a correct
orchestrate run.
'''
#ret_output = self.run_run_plus('state.orchestrate', 'orch.simple')['out']
ret_output = self.run_run('state.orchestrate orch.simple')
bad_out = ['outputter:', ' highstate']
good_out = [' Function: salt.state',
@ -64,11 +64,11 @@ class StateRunnerTest(ShellCase):
# First, check that we don't have the "bad" output that was displaying in
# Issue #31330 where only the highstate outputter was listed
self.assertIsNot(bad_out, ret_output)
assert bad_out != ret_output
# Now test that some expected good sample output is present in the return.
for item in good_out:
self.assertIn(item, ret_output)
assert item in ret_output
def test_orchestrate_nested(self):
'''
@ -81,9 +81,10 @@ class StateRunnerTest(ShellCase):
'state.orchestrate nested-orch.outer',
with_retcode=True)
self.assertFalse(os.path.exists('/tmp/ewu-2016-12-13'))
self.assertNotEqual(code, 0)
assert os.path.exists('/tmp/ewu-2016-12-13') is False
assert code != 0
@flaky
def test_orchestrate_target_exists(self):
'''
test orchestration when target exists
@ -96,8 +97,8 @@ class StateRunnerTest(ShellCase):
' Result: True']
second = [' ID: test-state',
' Function: salt.state',
' Result: True']
' Function: salt.state',
' Result: True']
third = [' ID: cmd.run',
' Function: salt.function',
@ -107,9 +108,10 @@ class StateRunnerTest(ShellCase):
for out in ret_out:
for item in out:
self.assertIn(item, ret)
assert item in ret
def test_orchestrate_target_doesnt_exists(self):
@flaky
def test_orchestrate_target_doesnt_exist(self):
'''
test orchestration when target doesn't exist
while using multiple states
@ -122,8 +124,8 @@ class StateRunnerTest(ShellCase):
' Result: False']
second = [' ID: test-state',
' Function: salt.state',
' Result: True']
' Function: salt.state',
' Result: True']
third = [' ID: cmd.run',
' Function: salt.function',
@ -133,7 +135,7 @@ class StateRunnerTest(ShellCase):
for out in ret_out:
for item in out:
self.assertIn(item, ret)
assert item in ret
def test_state_event(self):
'''
@ -151,7 +153,7 @@ class StateRunnerTest(ShellCase):
while q.empty():
self.run_salt('minion test.ping --static')
out = q.get()
self.assertIn(expect, str(out))
assert expect in str(out)
server_thread.join()
@ -164,9 +166,9 @@ class StateRunnerTest(ShellCase):
def count(thing, listobj):
return sum([obj.strip() == thing for obj in listobj])
self.assertEqual(count('ID: test subset', ret), 1)
self.assertEqual(count('Succeeded: 1', ret), 1)
self.assertEqual(count('Failed: 0', ret), 1)
assert count('ID: test subset', ret) == 1
assert count('Succeeded: 1', ret) == 1
assert count('Failed: 0', ret) == 1
def test_orchestrate_salt_function_return_false_failure(self):
'''
@ -184,10 +186,7 @@ class StateRunnerTest(ShellCase):
# Drill down to the changes dict
state_result = ret['data']['master']['salt_|-deploy_check_|-test.false_|-function']['result']
self.assertEqual(
state_result,
False,
)
assert state_result is False
@skipIf(salt.utils.is_windows(), '*NIX-only test')

8
tests/integration/shell/test_call.py
View file

@ -25,7 +25,7 @@ from tests.support.case import ShellCase
from tests.support.unit import skipIf
from tests.support.paths import FILES, TMP
from tests.support.mixins import ShellCaseCommonTestsMixin
from tests.support.helpers import destructiveTest
from tests.support.helpers import destructiveTest, flaky
from tests.integration.utils import testprogram
# Import salt libs
@ -106,12 +106,10 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin
log.debug('The pkg: {0} is already installed on the machine'.format(pkg))
@skipIf(sys.platform.startswith('win'), 'This test does not apply on Win')
@flaky
def test_user_delete_kw_output(self):
ret = self.run_call('-l quiet -d user.delete')
self.assertIn(
'salt \'*\' user.delete name remove=True force=True',
''.join(ret)
)
assert 'salt \'*\' user.delete name remove=True force=True' in ''.join(ret)
def test_salt_documentation_too_many_arguments(self):
'''

14
tests/integration/shell/test_matcher.py
View file

@ -11,8 +11,9 @@ import yaml
# Import Salt Testing libs
from tests.support.case import ShellCase
from tests.support.paths import TMP
from tests.support.helpers import flaky
from tests.support.mixins import ShellCaseCommonTestsMixin
from tests.support.paths import TMP
# Import salt libs
import salt.utils
@ -322,6 +323,7 @@ class MatchTest(ShellCase, ShellCaseCommonTestsMixin):
data = self.run_salt('-d "*" user')
self.assertIn('user.add:', data)
@flaky
def test_salt_documentation_arguments_not_assumed(self):
'''
Test to see if we're not auto-adding '*' and 'sys.doc' to the call
@ -333,16 +335,16 @@ class MatchTest(ShellCase, ShellCaseCommonTestsMixin):
'see https://github.com/saltstack/salt-jenkins/issues/324.')
data = self.run_salt('-d -t 20')
if data:
self.assertIn('user.add:', data)
assert 'user.add:' in data
data = self.run_salt('"*" -d -t 20')
if data:
self.assertIn('user.add:', data)
assert 'user.add:' in data
data = self.run_salt('"*" -d user -t 20')
self.assertIn('user.add:', data)
assert 'user.add:' in data
data = self.run_salt('"*" sys.doc -d user -t 20')
self.assertIn('user.add:', data)
assert 'user.add:' in data
data = self.run_salt('"*" sys.doc user -t 20')
self.assertIn('user.add:', data)
assert 'user.add:' in data
def test_salt_documentation_too_many_arguments(self):
'''