From 52b326d009545af9dbf44ec54e7ce9d610414517 Mon Sep 17 00:00:00 2001 From: Clément Pit--Claudel Date: Mon, 8 Jun 2015 18:14:53 -0700 Subject: Add the beginning of a new testing infrastructure runTests.py reads lit-style annotations, so we will be able to retain lit compatibility. This new framework adds: * Precise timings * Proper support for interrupting using Ctrl+C * Much better reporting (including tracking of error codes, and merging of successive reports for performance tracking) * No dependency on lit, OutputCheck, or Diff * Pretty colors! --- Test/runTests.bat | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Test/runTests.bat (limited to 'Test/runTests.bat') diff --git a/Test/runTests.bat b/Test/runTests.bat new file mode 100644 index 00000000..4eeed06b --- /dev/null +++ b/Test/runTests.bat @@ -0,0 +1,2 @@ +@REM runTests.bat -f "/dprelude PRELUDE_FILE" -r REPORT_NAME INPUT_FILES +C:/Python34/python.exe runTests.py --compiler "c:/MSR/dafny/Binaries/Dafny.exe /useBaseNameForFileName /compile:1 /nologo" --difftool "C:\Program Files (x86)\Meld\Meld.exe" -j4 %* -- cgit v1.2.3 From b9ed0285592f5f0d12d02b3d9155dfe46903b6c5 Mon Sep 17 00:00:00 2001 From: Clément Pit--Claudel Date: Wed, 10 Jun 2015 16:29:50 -0700 Subject: Small improvements to runTests.py --- Test/runTests.bat | 2 +- Test/runTests.py | 33 ++++++++++++++++++++------------- 2 files changed, 21 insertions(+), 14 deletions(-) (limited to 'Test/runTests.bat') diff --git a/Test/runTests.bat b/Test/runTests.bat index 4eeed06b..ced3bd27 100644 --- a/Test/runTests.bat +++ b/Test/runTests.bat @@ -1,2 +1,2 @@ @REM runTests.bat -f "/dprelude PRELUDE_FILE" -r REPORT_NAME INPUT_FILES -C:/Python34/python.exe runTests.py --compiler "c:/MSR/dafny/Binaries/Dafny.exe /useBaseNameForFileName /compile:1 /nologo" --difftool "C:\Program Files (x86)\Meld\Meld.exe" -j4 %* +C:/Python34/python.exe runTests.py --flags "/useBaseNameForFileName /compile:1 /nologo" --difftool "C:/Program Files (x86)/Meld/Meld.exe" %* diff --git a/Test/runTests.py b/Test/runTests.py index bf4b0d9d..1d16440e 100644 --- a/Test/runTests.py +++ b/Test/runTests.py @@ -137,11 +137,9 @@ class Test: @staticmethod def summarize(results): debug(None, "") - debug(Debug.INFO, "** Testing complete ({} tests) **".format(len(results))) + debug(Debug.INFO, "** Testing complete ({} tests) **".format(len(results))) if results: - debug(Debug.REPORT, "Testing took {:.2f}s on {} threads".format(results[0].suite_time, results[0].njobs)) - grouped = defaultdict(list) for t in results: grouped[t.status].append(t) @@ -149,6 +147,7 @@ class Test: for status, ts in sorted(grouped.items(), key=lambda x: x[0].index): if ts: debug(Debug.REPORT, "{}/{} -- {}".format(len(ts), len(results), ", ".join(t.name for t in ts)), headers=status) + debug(Debug.REPORT, "Testing took {:.2f}s on {} threads".format(results[0].suite_time, results[0].njobs)) def run(self): debug(Debug.DEBUG, "Starting {}".format(self.name)) @@ -224,11 +223,11 @@ def setup_parser(): parser.add_argument('paths', type=str, action='store', nargs='+', help='Input files or folders. Folders are searched for .dfy files.') - parser.add_argument('--compiler', type=str, action='store', default='Dafny.exe', - help='Command to use for %dafny.') + parser.add_argument('--compiler', type=str, action='store', default=None, + help='Dafny executable.') - parser.add_argument('--more-flags', '-f', type=str, action='store', default='', - help='Command used to run the tests.') + parser.add_argument('--flags', '-f', type=str, action='append', default=[], + help='Arguments to pass to dafny. Multiple --flags are concatenated.') parser.add_argument('--njobs', '-j', action='store', type=int, default=None, help='Number of test workers.') @@ -248,6 +247,9 @@ def setup_parser(): parser.add_argument('--compare', action='store_true', help="Compare two previously generated reports.") + parser.add_argument('--time-failures', action='store_true', + help="When comparing, include timings of failures.") + parser.add_argument('--diff', '-d', action='store_const', const=True, default=False, help="Don't run tests; show differences for one file.") @@ -336,14 +338,19 @@ def run_tests(args): debug(Debug.ERROR, "Compiler not found: {}".format(compiler_bin)) return - tests = list(find_tests(args.paths, args.compiler + ' ' + args.more_flags, + if args.compiler is None: + base_directory = os.path.dirname(os.path.realpath(__file__)) + compiler = os.path.normpath(os.path.join(base_directory, "../Binaries/Dafny.exe")) + + tests = list(find_tests(args.paths, compiler + ' ' + " ".join(args.flags), args.exclude + ALWAYS_EXCLUDED, args.timeout)) tests.sort(key=operator.attrgetter("name")) args.njobs = args.njobs or os.cpu_count() or 1 - debug(Debug.INFO, "** Running {} tests on {} testing threads, timeout is {:.2f} **".format(len(tests), args.njobs, args.timeout)) + debug(Debug.INFO, "** Running {} tests on {} testing threads, timeout is {:.2f}, started at {}**".format(len(tests), args.njobs, args.timeout, strftime("%H:%M:%S"))) + try: - pool = Pool(args.njobs) #, init_tester) + pool = Pool(args.njobs) start = time() results = [] @@ -378,7 +385,7 @@ def diff(paths, accept, difftool): else: call([difftool, test.expect_path, test.temp_output_path]) -def compare_results(globs): +def compare_results(globs, time_failures): from glob import glob paths = [path for g in globs for path in glob(g)] reports = {path: Test.load_report(path) for path in paths} @@ -404,7 +411,7 @@ def compare_results(globs): row.append(ref_duration) for path in paths[1:]: test_status, test_duration = resultsets[path][name] - if test_status == ref_status: + if test_status == ref_status or (test_status == TestStatus.FAILED and time_failures): result = "{:.2%}".format((test_duration - ref_duration) / ref_duration) else: result = test_status.name + "?!" @@ -449,7 +456,7 @@ def main(): elif args.open: os.startfile(args.paths[0]) elif args.compare: - compare_results(args.paths) + compare_results(args.paths, args.time_failures) else: run_tests(args) -- cgit v1.2.3 From c2092abbb945b55c87976d6d57ec2f728306af89 Mon Sep 17 00:00:00 2001 From: Clément Pit--Claudel Date: Thu, 23 Jul 2015 11:59:33 -0700 Subject: Let runTests.py generate expect files That is, missing expect files now raise a warning, not an error. --- Test/runTests.bat | 2 +- Test/runTests.py | 39 ++++++++++++++++++++++----------------- 2 files changed, 23 insertions(+), 18 deletions(-) (limited to 'Test/runTests.bat') diff --git a/Test/runTests.bat b/Test/runTests.bat index ced3bd27..4856faf8 100644 --- a/Test/runTests.bat +++ b/Test/runTests.bat @@ -1,2 +1,2 @@ @REM runTests.bat -f "/dprelude PRELUDE_FILE" -r REPORT_NAME INPUT_FILES -C:/Python34/python.exe runTests.py --flags "/useBaseNameForFileName /compile:1 /nologo" --difftool "C:/Program Files (x86)/Meld/Meld.exe" %* +C:/Python34/python.exe runTests.py --difftool "C:/Program Files (x86)/Meld/Meld.exe" %* diff --git a/Test/runTests.py b/Test/runTests.py index 02a6f039..34a9c4de 100644 --- a/Test/runTests.py +++ b/Test/runTests.py @@ -138,8 +138,12 @@ class Test: @staticmethod def read_normalize(path): - with open(path, mode="rb") as reader: - return reader.read().replace(b'\r\n', b'\n').replace(b'\r', b'\n') + try: + with open(path, mode="rb") as reader: + return reader.read().replace(b'\r\n', b'\n').replace(b'\r', b'\n') + except FileNotFoundError: + debug(Debug.WARNING, "{} not found".format(path)) + return "" @staticmethod def build_report(tests, name): @@ -232,12 +236,12 @@ class Test: def report(self, tid, running, alltests): running = [alltests[rid].fname for rid in running] - # running = ", ".join(running if len(running) <= 2 else (running[:2] + ["..."])) - running = "; oldest thread: {}".format(wrap_color(running[0], Colors.DIM)) if running else "" + running = "; oldest thread: {}".format(running[0]) if running else "" - fstring = "[{:5.2f}s] {} ({} of {}{})" + fstring = "[{:5.2f}s] {} ({}{})" + progress = "{} of {}".format(tid, len(alltests)) message = fstring.format(self.duration, wrap_color(self.dfy, Colors.BRIGHT), - tid, len(alltests), running) + wrap_color(progress, Colors.BRIGHT), running) debug(Debug.INFO, message, headers=self.status) @@ -295,14 +299,14 @@ def setup_parser(): help="When comparing, include all timings.") parser.add_argument('--diff', '-d', action='store_const', const=True, default=False, - help="Don't run tests; show differences for one file.") + help="Don't run tests; show differences between outputs and .expect files, optionally overwritting .expect files.") + + parser.add_argument('--accept', '-a', action='store_const', const=True, default=False, + help="Don't run tests; copy outputs to .expect files.") parser.add_argument('--open', '-o', action='store_const', const=True, default=False, help="Don't run tests; open one file.") - parser.add_argument('--accept', '-a', action='store_const', const=True, default=False, - help="Used in conjuction with --diff, accept the new output.") - parser.add_argument('--difftool', action='store', type=str, default="diff", help='Diff program. Default: diff.') @@ -346,10 +350,7 @@ def read_one_test(name, fname, compiler_cmds, timeout): else: break if cmds: - if os.path.exists(Test.source_to_expect_path(source_path)): - yield Test(name, source_path, cmds, timeout, cid) - else: - debug(Debug.DEBUG, "Test file {} has no .expect".format(fname)) + yield Test(name, source_path, cmds, timeout, cid) else: debug(Debug.INFO, "Test file {} has no RUN specification".format(fname)) @@ -436,12 +437,16 @@ def diff(paths, accept, difftool): debug(Debug.ERROR, "Not found: {}".format(path)) else: test = Test(None, path, [], None) + if not accept: call([difftool, test.expect_path, test.temp_output_path]) + accept = input("Accept this change? (y/N) ") == "y" - if accept or input("Accept this change? (y/N) ") == "y": - debug(Debug.INFO, path, "Accepted") + if accept: + debug(Debug.INFO, path, "accepted.") shutil.copy(test.temp_output_path, test.expect_path) + else: + debug(Debug.INFO, path, "not accepted.") def compare_results(globs, time_all): from glob import glob @@ -486,7 +491,7 @@ def main(): if os.name != 'nt' and os.environ.get("TERM") == "cygwin": debug(Debug.WARNING, "If you run into issues, try using Windows' Python instead of Cygwin's") - if args.diff: + if args.diff or args.accept: diff(args.paths, args.accept, args.difftool) elif args.open: os.startfile(args.paths[0]) -- cgit v1.2.3