From 1258fd132d80cfdba5e59cfd76c517a091269d2d Mon Sep 17 00:00:00 2001
From: Clément Pit--Claudel <clement.pitclaudel@live.com>
Date: Tue, 28 Jul 2015 17:56:40 -0700
Subject: Save failing tests to failing.lst, making it easy to re-run them

---
 Test/runTests.py | 36 +++++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 9 deletions(-)

(limited to 'Test/runTests.py')

diff --git a/Test/runTests.py b/Test/runTests.py
index 62ab205b..f9443426 100644
--- a/Test/runTests.py
+++ b/Test/runTests.py
@@ -184,6 +184,15 @@ class Test:
                         for test in tests:
                             debug(Debug.REPORT, "* " + test.dfy, headers=status, silentheaders=True)
 
+            debug(Debug.REPORT)
+
+            failing = [t for t in results if t.status != TestStatus.PASSED]
+            if failing:
+                with open("failing.lst", mode='w') as writer:
+                    for t in failing:
+                        writer.write("{}\t{}\n".format(t.name, t.source_path))
+                debug(Debug.REPORT, "Some tests failed: use [runTests.py failing.lst] to rerun the failing tests")
+
             debug(Debug.REPORT, "Testing took {:.2f}s on {} thread(s)".format(results[0].suite_time, results[0].njobs))
 
 
@@ -265,8 +274,8 @@ class Test:
 def setup_parser():
     parser = argparse.ArgumentParser(description='Run the Dafny test suite.')
 
-    parser.add_argument('paths', type=str, action='store', nargs='+',
-                        help='Input files or folders. Folders are searched for .dfy files.')
+    parser.add_argument('path', type=str, action='store', nargs='+',
+                        help='Input files or folders. Folders are searched for .dfy files. Lists of files can also be specified by passing a .lst file (for an example of such a file, look at failing.lst after running failing tests.')
 
     parser.add_argument('--compiler', type=str, action='append', default=None,
                         help='Dafny executable. Default: {}'.format(Defaults.DAFNY_BIN))
@@ -286,6 +295,9 @@ def setup_parser():
     parser.add_argument('--verbosity', action='store', type=int, default=1,
                         help='Set verbosity level. 0: Minimal; 1: Some info; 2: More info.')
 
+    parser.add_argument('-v', action='store_const', default=1, dest="verbosity", const=2,
+                        help='Short for --verbosity 2.')
+
     parser.add_argument('--report', '-r', action='store', type=str, default=None,
                         help='Give an explicit name to the report file. Defaults to the current date and time.')
 
@@ -298,13 +310,13 @@ def setup_parser():
     parser.add_argument('--time-all', action='store_true',
                         help="When comparing, include all timings.")
 
-    parser.add_argument('--diff', '-d', action='store_const', const=True, default=False,
+    parser.add_argument('--diff', '-d', action='store_true',
                         help="Don't run tests; show differences between outputs and .expect files, optionally overwritting .expect files.")
 
-    parser.add_argument('--accept', '-a', action='store_const', const=True, default=False,
+    parser.add_argument('--accept', '-a', action='store_true',
                         help="Don't run tests; copy outputs to .expect files.")
 
-    parser.add_argument('--open', '-o', action='store_const', const=True, default=False,
+    parser.add_argument('--open', '-o', action='store_true',
                         help="Don't run tests; open one file.")
 
     parser.add_argument('--difftool', action='store', type=str, default="diff",
@@ -363,6 +375,12 @@ def find_one(name, fname, compiler_cmds, timeout):
             yield from read_one_test(name, fname, compiler_cmds, timeout)
         else:
             debug(Debug.ERROR, "Test file {} not found".format(fname))
+    elif ext == ".lst":
+        debug(Debug.INFO, "Loading tests from {}".format(fname))
+        with open(fname) as reader:
+            for line in reader:
+                _name, _path = line.strip().split()
+                yield from find_one(_name, _path, compiler_cmds, timeout)
     else:
         debug(Debug.TRACE, "Ignoring {}".format(fname))
 
@@ -390,7 +408,7 @@ def run_tests(args):
             debug(Debug.ERROR, "Compiler not found: {}".format(compiler))
             return
 
-    tests = list(find_tests(args.paths, [compiler + ' ' + " ".join(args.base_flags + args.flags)
+    tests = list(find_tests(args.path, [compiler + ' ' + " ".join(args.base_flags + args.flags)
                                          for compiler in args.compiler],
                             args.exclude + Defaults.ALWAYS_EXCLUDED, args.timeout))
     tests.sort(key=operator.attrgetter("name"))
@@ -492,11 +510,11 @@ def main():
         debug(Debug.WARNING, "If you run into issues, try using Windows' Python instead of Cygwin's")
 
     if args.diff or args.accept:
-        diff(args.paths, args.accept, args.difftool)
+        diff(args.path, args.accept, args.difftool)
     elif args.open:
-        os.startfile(args.paths[0])
+        os.startfile(args.path[0])
     elif args.compare:
-        compare_results(args.paths, args.time_all)
+        compare_results(args.path, args.time_all)
     else:
         run_tests(args)
 
-- 
cgit v1.2.3