From b9ed0285592f5f0d12d02b3d9155dfe46903b6c5 Mon Sep 17 00:00:00 2001
From: Clément Pit--Claudel <clement.pitclaudel@live.com>
Date: Wed, 10 Jun 2015 16:29:50 -0700
Subject: Small improvements to runTests.py

---
 Test/runTests.py | 33 ++++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 13 deletions(-)

(limited to 'Test/runTests.py')

diff --git a/Test/runTests.py b/Test/runTests.py
index bf4b0d9d..1d16440e 100644
--- a/Test/runTests.py
+++ b/Test/runTests.py
@@ -137,11 +137,9 @@ class Test:
     @staticmethod
     def summarize(results):
         debug(None, "")
-        debug(Debug.INFO, "** Testing complete ({} tests) **".format(len(results)))
 
+        debug(Debug.INFO, "** Testing complete ({} tests) **".format(len(results)))
         if results:
-            debug(Debug.REPORT, "Testing took {:.2f}s on {} threads".format(results[0].suite_time, results[0].njobs))
-
             grouped = defaultdict(list)
             for t in results:
                 grouped[t.status].append(t)
@@ -149,6 +147,7 @@ class Test:
             for status, ts in sorted(grouped.items(), key=lambda x: x[0].index):
                 if ts:
                     debug(Debug.REPORT, "{}/{} -- {}".format(len(ts), len(results), ", ".join(t.name for t in ts)), headers=status)
+            debug(Debug.REPORT, "Testing took {:.2f}s on {} threads".format(results[0].suite_time, results[0].njobs))
 
     def run(self):
         debug(Debug.DEBUG, "Starting {}".format(self.name))
@@ -224,11 +223,11 @@ def setup_parser():
     parser.add_argument('paths', type=str, action='store', nargs='+',
                         help='Input files or folders. Folders are searched for .dfy files.')
 
-    parser.add_argument('--compiler', type=str, action='store', default='Dafny.exe',
-                        help='Command to use for %dafny.')
+    parser.add_argument('--compiler', type=str, action='store', default=None,
+                        help='Dafny executable.')
 
-    parser.add_argument('--more-flags', '-f', type=str, action='store', default='',
-                        help='Command used to run the tests.')
+    parser.add_argument('--flags', '-f', type=str, action='append', default=[],
+                        help='Arguments to pass to dafny. Multiple --flags are concatenated.')
 
     parser.add_argument('--njobs', '-j', action='store', type=int, default=None,
                         help='Number of test workers.')
@@ -248,6 +247,9 @@ def setup_parser():
     parser.add_argument('--compare', action='store_true',
                         help="Compare two previously generated reports.")
 
+    parser.add_argument('--time-failures', action='store_true',
+                        help="When comparing, include timings of failures.")
+
     parser.add_argument('--diff', '-d', action='store_const', const=True, default=False,
                         help="Don't run tests; show differences for one file.")
 
@@ -336,14 +338,19 @@ def run_tests(args):
         debug(Debug.ERROR, "Compiler not found: {}".format(compiler_bin))
         return
 
-    tests = list(find_tests(args.paths, args.compiler + ' ' + args.more_flags,
+    if args.compiler is None:
+        base_directory = os.path.dirname(os.path.realpath(__file__))
+        compiler = os.path.normpath(os.path.join(base_directory, "../Binaries/Dafny.exe"))
+
+    tests = list(find_tests(args.paths, compiler + ' ' + " ".join(args.flags),
                             args.exclude + ALWAYS_EXCLUDED, args.timeout))
     tests.sort(key=operator.attrgetter("name"))
 
     args.njobs = args.njobs or os.cpu_count() or 1
-    debug(Debug.INFO, "** Running {} tests on {} testing threads, timeout is {:.2f} **".format(len(tests), args.njobs, args.timeout))
+    debug(Debug.INFO, "** Running {} tests on {} testing threads, timeout is {:.2f}, started at {}**".format(len(tests), args.njobs, args.timeout, strftime("%H:%M:%S")))
+
     try:
-        pool = Pool(args.njobs) #, init_tester)
+        pool = Pool(args.njobs)
 
         start = time()
         results = []
@@ -378,7 +385,7 @@ def diff(paths, accept, difftool):
         else:
             call([difftool, test.expect_path, test.temp_output_path])
 
-def compare_results(globs):
+def compare_results(globs, time_failures):
     from glob import glob
     paths = [path for g in globs for path in glob(g)]
     reports = {path: Test.load_report(path) for path in paths}
@@ -404,7 +411,7 @@ def compare_results(globs):
             row.append(ref_duration)
             for path in paths[1:]:
                 test_status, test_duration = resultsets[path][name]
-                if test_status == ref_status:
+                if test_status == ref_status or (test_status == TestStatus.FAILED and time_failures):
                     result = "{:.2%}".format((test_duration - ref_duration) / ref_duration)
                 else:
                     result = test_status.name + "?!"
@@ -449,7 +456,7 @@ def main():
     elif args.open:
         os.startfile(args.paths[0])
     elif args.compare:
-        compare_results(args.paths)
+        compare_results(args.paths, args.time_failures)
     else:
         run_tests(args)
 
-- 
cgit v1.2.3