Rerun failed tests

noxfile.py

@@ -21,6 +21,7 @@ if __name__ == '__main__':
 
 # Import 3rd-party libs
 import nox
+from nox.command import CommandFailed
 
 # Global Path Definitions
 REPO_ROOT = os.path.abspath(os.path.dirname(__file__))
@@ -165,10 +166,46 @@ def runtests(session, coverage):
         )
     ] + session.posargs
 
-    if coverage is True:
-        _run_with_coverage(session, 'coverage', 'run', '-m', 'tests.runtests', *cmd_args)
-    else:
-        session.run('python', os.path.join('tests', 'runtests.py'), *cmd_args)
+    try:
+        if coverage is True:
+            _run_with_coverage(session, 'coverage', 'run', '-m', 'tests.runtests', *cmd_args)
+        else:
+            session.run('python', os.path.join('tests', 'runtests.py'), *cmd_args)
+    except CommandFailed:
+        session.log('Re-running failed tests if possible')
+        names_file_path = os.path.join('artifacts', 'failed-tests.txt')
+        session.install('xunitparser==1.3.3')
+        session.run(
+            'python',
+            os.path.join('tests', 'support', 'generate-names-file-from-failed-test-reports.py'),
+            names_file_path
+        )
+        if not os.path.exists(names_file_path):
+            session.error('No names file was generated to re-run tests')
+
+        with open(names_file_path) as rfh:
+            failed_tests_count = len(rfh.read().splitlines())
+            if failed_tests_count > 500:
+                # 500 test failures?! Something else must have gone wrong, don't even bother
+                session.error(
+                    'Total failed tests({}) > 500. No point on re-running the failed tests'.format(
+                        failed_tests_count
+                    )
+                )
+
+        for idx, flag in enumerate(cmd_args[:]):
+            if '--names-file=' in flag:
+                cmd_args.pop(idx)
+                break
+            elif flag == '--names-file':
+                cmd_args.pop(idx)  # pop --names-file
+                cmd_args.pop(idx)  # pop the actual names file
+                break
+        cmd_args.append('--names-file={}'.format(names_file_path))
+        if coverage is True:
+            _run_with_coverage(session, 'coverage', 'run', '-m', 'tests.runtests', *cmd_args)
+        else:
+            session.run('python', os.path.join('tests', 'runtests.py'), *cmd_args)
 
 
 @nox.session(python=_PYTHON_VERSIONS)
@@ -189,7 +226,16 @@ def pytest(session, coverage):
         '-s'
     ] + session.posargs
 
-    if coverage is True:
-        _run_with_coverage(session, 'coverage', 'run', '-m', 'py.test', *cmd_args)
-    else:
-        session.run('py.test', *cmd_args)
+    try:
+        if coverage is True:
+            _run_with_coverage(session, 'coverage', 'run', '-m', 'py.test', *cmd_args)
+        else:
+            session.run('py.test', *cmd_args)
+    except CommandFailed:
+        # Re-run failed tests
+        session.log('Re-running failed tests')
+        cmd_args.append('--lf')
+        if coverage is True:
+            _run_with_coverage(session, 'coverage', 'run', '-m', 'py.test', *cmd_args)
+        else:
+            session.run('py.test', *cmd_args)

tests/support/generate-names-file-from-failed-test-reports.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+'''
+    tests.support.generate-from-names-from-failed-test-reports
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    This script is meant as a stop-gap until we move to PyTest to provide a functionality similar to
+    PyTest's --last-failed where PyTest only runs last failed tests.
+'''
+
+from __future__ import absolute_import, unicode_literals, print_function
+import os
+import sys
+import glob
+import argparse
+
+try:
+    import xunitparser
+except ImportError:
+    sys.stderr.write('Please install the xunitparser python package to run this script\n')
+    sys.stderr.flush()
+    sys.exit(1)
+
+REPO_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--reports-dir',
+        default=os.path.join(REPO_ROOT, 'artifacts', 'xml-unittests-output'),
+        help='Path to the directory where the JUnit XML reports can be found'
+    )
+    parser.add_argument(
+        'output_file',
+        help='Path to the file containing the failed tests listing to be fed to --names-files'
+    )
+    options = parser.parse_args()
+    total_xml_reports = 0
+    failures = set()
+    for fname in sorted(glob.glob(os.path.join(options.reports_dir, '*.xml'))):
+        total_xml_reports += 1
+        with open(fname) as rfh:
+            test_suite, test_result = xunitparser.parse(rfh)
+            if not test_result.errors and not test_result.failures:
+                continue
+            for test in test_suite:
+                if test.bad:
+                    failures.add('{classname}.{methodname}'.format(**test.__dict__))
+
+    if not total_xml_reports:
+        parser.exit(status=1, message='No JUnit XML files were parsed')
+
+    with open(options.output_file, 'w') as wfh:
+        wfh.write(os.linesep.join(sorted(failures)))
+
+    parser.exit(status=0)
+
+
+if __name__ == '__main__':
+    main()