diff options
13 files changed, 272 insertions, 379 deletions
diff --git a/gm/gm_expectations.cpp b/gm/gm_expectations.cpp index ff6fc2c70f..9c0c274d3f 100644 --- a/gm/gm_expectations.cpp +++ b/gm/gm_expectations.cpp @@ -11,14 +11,13 @@ #define DEBUGFAIL_SEE_STDERR SkDEBUGFAIL("see stderr for message") -// These constants must be kept in sync with the JSONKEY_ constants in -// gm_json.py ! +// See gm_json.py for descriptions of each of these JSON keys. +// These constants must be kept in sync with the ones in that Python file! const static char kJsonKey_ActualResults[] = "actual-results"; const static char kJsonKey_ActualResults_Failed[] = "failed"; const static char kJsonKey_ActualResults_FailureIgnored[]= "failure-ignored"; const static char kJsonKey_ActualResults_NoComparison[] = "no-comparison"; const static char kJsonKey_ActualResults_Succeeded[] = "succeeded"; - const static char kJsonKey_ExpectedResults[] = "expected-results"; const static char kJsonKey_ExpectedResults_AllowedDigests[] = "allowed-digests"; const static char kJsonKey_ExpectedResults_IgnoreFailure[] = "ignore-failure"; diff --git a/gm/gm_json.py b/gm/gm_json.py index cd4198415c..35b954e6ae 100644 --- a/gm/gm_json.py +++ b/gm/gm_json.py @@ -15,18 +15,38 @@ __author__ = 'Elliot Poger' import json +# Key strings used in GM results JSON files (both expected-results.json and +# actual-results.json). +# # These constants must be kept in sync with the kJsonKey_ constants in # gm_expectations.cpp ! + JSONKEY_ACTUALRESULTS = 'actual-results' +# Tests whose results failed to match expectations. JSONKEY_ACTUALRESULTS_FAILED = 'failed' +# Tests whose results failed to match expectations, but IGNOREFAILURE causes +# us to take them less seriously. JSONKEY_ACTUALRESULTS_FAILUREIGNORED = 'failure-ignored' +# Tests for which we do not have any expectations. They may be new tests that +# we haven't had a chance to check in expectations for yet, or we may have +# consciously decided to leave them without expectations because we are unhappy +# with the results (although we should try to move away from that, and instead +# check in expectations with the IGNOREFAILURE flag set). JSONKEY_ACTUALRESULTS_NOCOMPARISON = 'no-comparison' +# Tests whose results matched their expectations. JSONKEY_ACTUALRESULTS_SUCCEEDED = 'succeeded' JSONKEY_EXPECTEDRESULTS = 'expected-results' +# One or more [HashType/DigestValue] pairs representing valid results for this +# test. Typically, there will just be one pair, but we allow for multiple +# expectations, and the test will pass if any one of them is matched. JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS = 'allowed-digests' +# If IGNOREFAILURE is set to True, a failure of this test will be reported +# within the FAILUREIGNORED section (thus NOT causing the buildbots to go red) +# rather than the FAILED section (which WOULD cause the buildbots to go red). JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE = 'ignore-failure' +# Allowed hash types for test expectations. JSONKEY_HASHTYPE_BITMAP_64BITMD5 = 'bitmap-64bitMD5' def LoadFromString(file_contents): @@ -45,3 +65,8 @@ def LoadFromFile(file_path): above.""" file_contents = open(file_path, 'r').read() return LoadFromString(file_contents) + +def WriteToFile(json_dict, file_path): + """Writes the JSON summary in json_dict out to file_path.""" + with open(file_path, 'w') as outfile: + json.dump(json_dict, outfile, sort_keys=True, indent=2) diff --git a/tools/rebaseline.py b/tools/rebaseline.py index a1ea6996c3..40d6c61444 100755 --- a/tools/rebaseline.py +++ b/tools/rebaseline.py @@ -17,7 +17,6 @@ checkout, the files will be added to the staging area for commit. import argparse import os import re -import subprocess import sys import urllib2 @@ -41,8 +40,6 @@ if GM_DIRECTORY not in sys.path: sys.path.append(GM_DIRECTORY) import gm_json -JSON_EXPECTATIONS_FILENAME='expected-results.json' - # Mapping of gm-expectations subdir (under # https://skia.googlecode.com/svn/gm-expected/ ) # to builder name (see list at http://108.170.217.252:10117/builders ) @@ -78,130 +75,45 @@ class CommandFailedException(Exception): pass # Object that rebaselines a JSON expectations file (not individual image files). -# -# TODO(epoger): Most of this is just the code from the old ImageRebaseliner... -# some of it will need to be updated in order to properly rebaseline JSON files. -# There is a lot of code duplicated between here and ImageRebaseliner, but -# that's fine because we will delete ImageRebaseliner soon. class JsonRebaseliner(object): # params: - # expectations_root: root directory of all expectations - # json_base_url: base URL from which to read json_filename - # json_filename: filename (under json_base_url) from which to read a - # summary of results; typically "actual-results.json" + # expectations_root: root directory of all expectations JSON files + # expectations_filename: filename (under expectations_root) of JSON + # expectations file; typically + # "expected-results.json" + # actuals_base_url: base URL from which to read actual-result JSON files + # actuals_filename: filename (under actuals_base_url) from which to read a + # summary of results; typically "actual-results.json" # tests: list of tests to rebaseline, or None if we should rebaseline # whatever files the JSON results summary file tells us to # configs: which configs to run for each test; this should only be # specified if the list of tests was also specified (otherwise, # the JSON file will give us test names and configs) - # dry_run: if True, instead of actually downloading files or adding - # files to checkout, display a list of operations that - # we would normally perform # add_new: if True, add expectations for tests which don't have any yet - # missing_json_is_fatal: whether to halt execution if we cannot read a - # JSON actual result summary file - def __init__(self, expectations_root, json_base_url, json_filename, - tests=None, configs=None, dry_run=False, - add_new=False, missing_json_is_fatal=False): - raise ValueError('JsonRebaseliner not yet implemented') # TODO(epoger) + def __init__(self, expectations_root, expectations_filename, + actuals_base_url, actuals_filename, + tests=None, configs=None, add_new=False): if configs and not tests: raise ValueError('configs should only be specified if tests ' + 'were specified also') self._expectations_root = expectations_root + self._expectations_filename = expectations_filename self._tests = tests self._configs = configs - self._json_base_url = json_base_url - self._json_filename = json_filename - self._dry_run = dry_run + self._actuals_base_url = actuals_base_url + self._actuals_filename = actuals_filename self._add_new = add_new - self._missing_json_is_fatal = missing_json_is_fatal - self._googlestorage_gm_actuals_root = ( - 'http://chromium-skia-gm.commondatastorage.googleapis.com/gm') self._testname_pattern = re.compile('(\S+)_(\S+).png') - self._is_svn_checkout = ( - os.path.exists('.svn') or - os.path.exists(os.path.join(os.pardir, '.svn'))) - self._is_git_checkout = ( - os.path.exists('.git') or - os.path.exists(os.path.join(os.pardir, '.git'))) - - # If dry_run is False, execute subprocess.call(cmd). - # If dry_run is True, print the command we would have otherwise run. - # Raises a CommandFailedException if the command fails. - def _Call(self, cmd): - if self._dry_run: - print '%s' % ' '.join(cmd) - return - if subprocess.call(cmd) != 0: - raise CommandFailedException('error running command: ' + - ' '.join(cmd)) - - # Download a single actual result from GoogleStorage, returning True if it - # succeeded. - def _DownloadFromGoogleStorage(self, infilename, outfilename, all_results): - test_name = self._testname_pattern.match(infilename).group(1) - if not test_name: - print '# unable to find test_name for infilename %s' % infilename - return False - try: - hash_type, hash_value = all_results[infilename] - except KeyError: - print ('# unable to find filename %s in all_results dict' % - infilename) - return False - except ValueError as e: - print '# ValueError reading filename %s from all_results dict: %s'%( - infilename, e) - return False - url = '%s/%s/%s/%s.png' % (self._googlestorage_gm_actuals_root, - hash_type, test_name, hash_value) - try: - self._DownloadFile(source_url=url, dest_filename=outfilename) - return True - except CommandFailedException: - print '# Couldn\'t fetch gs_url %s' % url - return False - - # Download a single actual result from skia-autogen, returning True if it - # succeeded. - def _DownloadFromAutogen(self, infilename, outfilename, - expectations_subdir, builder_name): - url = ('http://skia-autogen.googlecode.com/svn/gm-actual/' + - expectations_subdir + '/' + builder_name + '/' + - expectations_subdir + '/' + infilename) - try: - self._DownloadFile(source_url=url, dest_filename=outfilename) - return True - except CommandFailedException: - print '# Couldn\'t fetch autogen_url %s' % url - return False - # Download a single file, raising a CommandFailedException if it fails. - def _DownloadFile(self, source_url, dest_filename): - # Download into a temporary file and then rename it afterwards, - # so that we don't corrupt the existing file if it fails midway thru. - temp_filename = os.path.join(os.path.dirname(dest_filename), - '.temp-' + os.path.basename(dest_filename)) - - # TODO(epoger): Replace calls to "curl"/"mv" (which will only work on - # Unix) with a Python HTTP library (which should work cross-platform) - self._Call([ 'curl', '--fail', '--silent', source_url, - '--output', temp_filename ]) - self._Call([ 'mv', temp_filename, dest_filename ]) - - # Returns the full contents of a URL, as a single string. - # - # Unlike standard URL handling, we allow relative "file:" URLs; - # for example, "file:one/two" resolves to the file ./one/two - # (relative to current working dir) - def _GetContentsOfUrl(self, url): - file_prefix = 'file:' - if url.startswith(file_prefix): - filename = url[len(file_prefix):] - return open(filename, 'r').read() + # Returns the full contents of filepath, as a single string. + # If filepath looks like a URL, try to read it that way instead of as + # a path on local storage. + def _GetFileContents(self, filepath): + if filepath.startswith('http:') or filepath.startswith('https:'): + return urllib2.urlopen(filepath).read() else: - return urllib2.urlopen(url).read() + return open(filepath, 'r').read() # Returns a dictionary of actual results from actual-results.json file. # @@ -212,10 +124,8 @@ class JsonRebaseliner(object): # u'shadertext3_8888.png': [u'bitmap-64bitMD5', 3713708307125704716] # } # - # If the JSON actual result summary file cannot be loaded, the behavior - # depends on self._missing_json_is_fatal: - # - if true: execution will halt with an exception - # - if false: we will log an error message but return an empty dictionary + # If the JSON actual result summary file cannot be loaded, raise an + # exception. # # params: # json_url: URL pointing to a JSON actual result summary file @@ -224,16 +134,7 @@ class JsonRebaseliner(object): # gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON] ; # if None, then include ALL sections. def _GetActualResults(self, json_url, sections=None): - try: - json_contents = self._GetContentsOfUrl(json_url) - except (urllib2.HTTPError, IOError): - message = 'unable to load JSON summary URL %s' % json_url - if self._missing_json_is_fatal: - raise ValueError(message) - else: - print '# %s' % message - return {} - + json_contents = self._GetFileContents(json_url) json_dict = gm_json.LoadFromString(json_contents) results_to_return = {} actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS] @@ -245,172 +146,108 @@ class JsonRebaseliner(object): results_to_return.update(section_results) return results_to_return - # Returns a list of files that require rebaselining. - # - # Note that this returns a list of FILES, like this: - # ['imageblur_565.png', 'xfermodes_pdf.png'] - # rather than a list of TESTS, like this: - # ['imageblur', 'xfermodes'] + # Rebaseline all tests/types we specified in the constructor, + # within this gm-expectations subdir. # # params: - # json_url: URL pointing to a JSON actual result summary file - # add_new: if True, then return files listed in any of these sections: - # - JSONKEY_ACTUALRESULTS_FAILED - # - JSONKEY_ACTUALRESULTS_NOCOMPARISON - # if False, then return files listed in these sections: - # - JSONKEY_ACTUALRESULTS_FAILED - # - def _GetFilesToRebaseline(self, json_url, add_new): - if self._dry_run: - print '' - print '#' - print ('# Getting files to rebaseline from JSON summary URL %s ...' - % json_url) + # subdir : e.g. 'base-shuttle-win7-intel-float' + # builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release' + def RebaselineSubdir(self, subdir, builder): + # Read in the actual result summary, and extract all the tests whose + # results we need to update. + actuals_url = '/'.join([self._actuals_base_url, + subdir, builder, subdir, + self._actuals_filename]) sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED] - if add_new: + if self._add_new: sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON) - results_to_rebaseline = self._GetActualResults(json_url=json_url, - sections=sections) - files_to_rebaseline = results_to_rebaseline.keys() - files_to_rebaseline.sort() - print '# ... found files_to_rebaseline %s' % files_to_rebaseline - if self._dry_run: - print '#' - return files_to_rebaseline + results_to_update = self._GetActualResults(json_url=actuals_url, + sections=sections) - # Rebaseline a single file. - def _RebaselineOneFile(self, expectations_subdir, builder_name, - infilename, outfilename, all_results): - if self._dry_run: - print '' - print '# ' + infilename + # Read in current expectations. + expectations_json_filepath = os.path.join( + self._expectations_root, subdir, self._expectations_filename) + expectations_dict = gm_json.LoadFromFile(expectations_json_filepath) - # First try to download this result image from Google Storage. - # If that fails, try skia-autogen. - # If that fails too, just go on to the next file. - # - # This not treated as a fatal failure because not all - # platforms generate all configs (e.g., Android does not - # generate PDF). - # - # TODO(epoger): Once we are downloading only files that the - # actual-results.json file told us to, this should become a - # fatal error. (If the actual-results.json file told us that - # the test failed with XXX results, we should be able to download - # those results every time.) - if not self._DownloadFromGoogleStorage(infilename=infilename, - outfilename=outfilename, - all_results=all_results): - if not self._DownloadFromAutogen(infilename=infilename, - outfilename=outfilename, - expectations_subdir=expectations_subdir, - builder_name=builder_name): - print '# Couldn\'t fetch infilename ' + infilename - return + # Update the expectations in memory, skipping any tests/configs that + # the caller asked to exclude. + skipped_images = [] + if results_to_update: + for (image_name, image_results) in results_to_update.iteritems(): + (test, config) = self._testname_pattern.match(image_name).groups() + if self._tests: + if test not in self._tests: + skipped_images.append(image_name) + continue + if self._configs: + if config not in self._configs: + skipped_images.append(image_name) + continue + expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] \ + [image_name] \ + [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS] = \ + [image_results] - # Add this file to version control (if appropriate). - if self._add_new: - if self._is_svn_checkout: - cmd = [ 'svn', 'add', '--quiet', outfilename ] - self._Call(cmd) - cmd = [ 'svn', 'propset', '--quiet', 'svn:mime-type', - 'image/png', outfilename ]; - self._Call(cmd) - elif self._is_git_checkout: - cmd = [ 'git', 'add', outfilename ] - self._Call(cmd) + # Write out updated expectations. + gm_json.WriteToFile(expectations_dict, expectations_json_filepath) - # Rebaseline the given configs for a single test. - # - # params: - # expectations_subdir - # builder_name - # test: a single test to rebaseline - # all_results: a dictionary of all actual results - def _RebaselineOneTest(self, expectations_subdir, builder_name, test, - all_results): - if self._configs: - configs = self._configs - else: - if (expectations_subdir == 'base-shuttle-win7-intel-angle'): - configs = [ 'angle', 'anglemsaa16' ] - else: - configs = [ '565', '8888', 'gpu', 'pdf', 'mesa', 'msaa16', - 'msaa4' ] - if self._dry_run: - print '' - print '# ' + expectations_subdir + ':' - for config in configs: - infilename = test + '_' + config + '.png' - outfilename = os.path.join(expectations_subdir, infilename); - self._RebaselineOneFile(expectations_subdir=expectations_subdir, - builder_name=builder_name, - infilename=infilename, - outfilename=outfilename, - all_results=all_results) - - # Rebaseline all tests/types we specified in the constructor, - # within this gm-expectations subdir. - # - # params: - # subdir : e.g. 'base-shuttle-win7-intel-float' - # builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release' - def RebaselineSubdir(self, subdir, builder): - json_url = '/'.join([self._json_base_url, - subdir, builder, subdir, - self._json_filename]) - all_results = self._GetActualResults(json_url=json_url) + if skipped_images: + print ('Skipped these tests due to test/config filters: %s' % + skipped_images) - if self._tests: - for test in self._tests: - self._RebaselineOneTest(expectations_subdir=subdir, - builder_name=builder, - test=test, all_results=all_results) - else: # get the raw list of files that need rebaselining from JSON - filenames = self._GetFilesToRebaseline(json_url=json_url, - add_new=self._add_new) - for filename in filenames: - outfilename = os.path.join(subdir, filename); - self._RebaselineOneFile(expectations_subdir=subdir, - builder_name=builder, - infilename=filename, - outfilename=outfilename, - all_results=all_results) # main... parser = argparse.ArgumentParser() +parser.add_argument('--actuals-base-url', + help='base URL from which to read files containing JSON ' + + 'summaries of actual GM results; defaults to %(default)s', + default='http://skia-autogen.googlecode.com/svn/gm-actual') +parser.add_argument('--actuals-filename', + help='filename (within platform-specific subdirectories ' + + 'of ACTUALS_BASE_URL) to read a summary of results from; ' + + 'defaults to %(default)s', + default='actual-results.json') +# TODO(epoger): Add test that exercises --add-new argument. parser.add_argument('--add-new', action='store_true', help='in addition to the standard behavior of ' + 'updating expectations for failing tests, add ' + 'expectations for tests which don\'t have expectations ' + 'yet.') +# TODO(epoger): Add test that exercises --configs argument. +# TODO(epoger): Once we are only rebaselining JSON files, update the helpstring +# to indicate that this is a *filter* over the config names that +# actual-results.json tells us need to be rebaselined. +# You don't need to specify tests also, etc. parser.add_argument('--configs', metavar='CONFIG', nargs='+', help='which configurations to rebaseline, e.g. ' + '"--configs 565 8888"; if unspecified, run a default ' + 'set of configs. This should ONLY be specified if ' + '--tests has also been specified.') +# TODO(epoger): The --dry-run argument will no longer be needed once we +# are only rebaselining JSON files. parser.add_argument('--dry-run', action='store_true', help='instead of actually downloading files or adding ' + 'files to checkout, display a list of operations that ' + 'we would normally perform') +parser.add_argument('--expectations-filename', + help='filename (under EXPECTATIONS_ROOT) to read ' + + 'current expectations from, and to write new ' + + 'expectations into; defaults to %(default)s', + default='expected-results.json') parser.add_argument('--expectations-root', help='root of expectations directory to update-- should ' + 'contain one or more base-* subdirectories. Defaults to ' + '%(default)s', default='.') -parser.add_argument('--json-base-url', - help='base URL from which to read JSON_FILENAME ' + - 'files; defaults to %(default)s', - default='http://skia-autogen.googlecode.com/svn/gm-actual') -parser.add_argument('--json-filename', - help='filename (under JSON_BASE_URL) to read a summary ' + - 'of results from; defaults to %(default)s', - default='actual-results.json') parser.add_argument('--subdirs', metavar='SUBDIR', nargs='+', help='which platform subdirectories to rebaseline; ' + 'if unspecified, rebaseline all subdirs, same as ' + '"--subdirs %s"' % ' '.join(sorted(SUBDIR_MAPPING.keys()))) +# TODO(epoger): Add test that exercises --tests argument. +# TODO(epoger): Once we are only rebaselining JSON files, update the helpstring +# to indicate that this is a *filter* over the test names that +# actual-results.json tells us need to be rebaselined. parser.add_argument('--tests', metavar='TEST', nargs='+', help='which tests to rebaseline, e.g. ' + '"--tests aaclip bigmatrix"; if unspecified, then all ' + @@ -438,25 +275,22 @@ for subdir in subdirs: # # See https://goto.google.com/ChecksumTransitionDetail expectations_json_file = os.path.join(args.expectations_root, subdir, - JSON_EXPECTATIONS_FILENAME) + args.expectations_filename) if os.path.isfile(expectations_json_file): - sys.stderr.write('ERROR: JsonRebaseliner is not implemented yet.\n') - sys.exit(1) rebaseliner = JsonRebaseliner( expectations_root=args.expectations_root, + expectations_filename=args.expectations_filename, tests=args.tests, configs=args.configs, - dry_run=args.dry_run, - json_base_url=args.json_base_url, - json_filename=args.json_filename, - add_new=args.add_new, - missing_json_is_fatal=missing_json_is_fatal) + actuals_base_url=args.actuals_base_url, + actuals_filename=args.actuals_filename, + add_new=args.add_new) else: rebaseliner = rebaseline_imagefiles.ImageRebaseliner( expectations_root=args.expectations_root, tests=args.tests, configs=args.configs, dry_run=args.dry_run, - json_base_url=args.json_base_url, - json_filename=args.json_filename, + json_base_url=args.actuals_base_url, + json_filename=args.actuals_filename, add_new=args.add_new, missing_json_is_fatal=missing_json_is_fatal) rebaseliner.RebaselineSubdir(subdir=subdir, builder=builder) diff --git a/tools/tests/rebaseline/output/all/output-expected/command_line b/tools/tests/rebaseline/output/all/output-expected/command_line index f51a2b2bbf..ac35415e1a 100644 --- a/tools/tests/rebaseline/output/all/output-expected/command_line +++ b/tools/tests/rebaseline/output/all/output-expected/command_line @@ -1 +1 @@ -python tools/rebaseline.py --dry-run --json-base-url file:nonexistent-path --tests test1 test2 +python tools/rebaseline.py --dry-run --actuals-base-url file:nonexistent-path --tests test1 test2 diff --git a/tools/tests/rebaseline/output/subset/output-expected/command_line b/tools/tests/rebaseline/output/subset/output-expected/command_line index ac278d2fc0..d66dc01761 100644 --- a/tools/tests/rebaseline/output/subset/output-expected/command_line +++ b/tools/tests/rebaseline/output/subset/output-expected/command_line @@ -1 +1 @@ -python tools/rebaseline.py --dry-run --expectations-root fake/expectations/path --json-base-url file:tools/tests/rebaseline/input/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float +python tools/rebaseline.py --dry-run --expectations-root fake/expectations/path --actuals-base-url file:tools/tests/rebaseline/input/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float diff --git a/tools/tests/rebaseline/output/using-json1-add-new/output-expected/command_line b/tools/tests/rebaseline/output/using-json1-add-new/output-expected/command_line index 32bd0ef9e7..9bd9539da6 100644 --- a/tools/tests/rebaseline/output/using-json1-add-new/output-expected/command_line +++ b/tools/tests/rebaseline/output/using-json1-add-new/output-expected/command_line @@ -1 +1 @@ -python tools/rebaseline.py --dry-run --json-base-url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --add-new +python tools/rebaseline.py --dry-run --actuals-base-url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --add-new diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line index 76293588ef..ac6be7dbd5 100644 --- a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line +++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/command_line @@ -1 +1 @@ -python tools/rebaseline.py --expectations-root tools/tests/rebaseline/output/using-json1-expectations/output-actual/gm-expectations --json-base-url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float +python tools/rebaseline.py --expectations-root tools/tests/rebaseline/output/using-json1-expectations/output-actual/gm-expectations --actuals-base-url tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-android-galaxy-nexus/expected-results.json b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-android-galaxy-nexus/expected-results.json index d7608cd2a0..20721f3525 100644 --- a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-android-galaxy-nexus/expected-results.json +++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-android-galaxy-nexus/expected-results.json @@ -1,52 +1,70 @@ { - "expected-results" : { - "3x3bitmaprect_565.png" : { - "allowed-digests" : null, - "ignore-failure" : false - }, - "3x3bitmaprect_8888.png" : { - "allowed-digests" : null, - "ignore-failure" : false - }, - "aaclip_gpu.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 11899819492385205974 ] - ], - "ignore-failure" : false - }, - "aarectmodes_565.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 14760033689012826769 ] - ], - "ignore-failure" : false - }, - "imageblur_565.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 17796243856503591523 ] - ], - "ignore-failure" : false - }, - "imageblur_8888.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 7426416989687670152 ] - ], - "ignore-failure" : false - }, - "shadertext3_8888.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 10593797161686785561 ] - ], - "ignore-failure" : false - }, - "xfermodeimagefilter_pdf.png" : { - "allowed-digests" : null, - "ignore-failure" : false - }, - "xfermodes_pdf.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 9151974350149210736 ] - ], - "ignore-failure" : false - } - } -} + "expected-results": { + "3x3bitmaprect_565.png": { + "allowed-digests": null, + "ignore-failure": false + }, + "3x3bitmaprect_8888.png": { + "allowed-digests": null, + "ignore-failure": false + }, + "aaclip_gpu.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 11899819492385205974 + ] + ], + "ignore-failure": false + }, + "aarectmodes_565.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 14760033689012826769 + ] + ], + "ignore-failure": false + }, + "imageblur_565.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 3359963596899141322 + ] + ], + "ignore-failure": false + }, + "imageblur_8888.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 4217923806027861152 + ] + ], + "ignore-failure": false + }, + "shadertext3_8888.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 3713708307125704716 + ] + ], + "ignore-failure": false + }, + "xfermodeimagefilter_pdf.png": { + "allowed-digests": null, + "ignore-failure": false + }, + "xfermodes_pdf.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 9151974350149210736 + ] + ], + "ignore-failure": false + } + } +}
\ No newline at end of file diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-shuttle-win7-intel-float/expected-results.json b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-shuttle-win7-intel-float/expected-results.json index b314ed68a4..20721f3525 100644 --- a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-shuttle-win7-intel-float/expected-results.json +++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/gm-expectations/base-shuttle-win7-intel-float/expected-results.json @@ -1,52 +1,70 @@ { - "expected-results" : { - "3x3bitmaprect_565.png" : { - "allowed-digests" : null, - "ignore-failure" : false - }, - "3x3bitmaprect_8888.png" : { - "allowed-digests" : null, - "ignore-failure" : false - }, - "aaclip_gpu.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 11899819492385205974 ] - ], - "ignore-failure" : false - }, - "aarectmodes_565.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 14760033689012826769 ] - ], - "ignore-failure" : false - }, - "imageblur_565.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 3359963596899141322 ] - ], - "ignore-failure" : false - }, - "imageblur_8888.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 4217923806027861152 ] - ], - "ignore-failure" : false - }, - "shadertext3_8888.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 3713708307125704716 ] - ], - "ignore-failure" : false - }, - "xfermodeimagefilter_pdf.png" : { - "allowed-digests" : null, - "ignore-failure" : false - }, - "xfermodes_pdf.png" : { - "allowed-digests" : [ - [ "bitmap-64bitMD5", 9151974350149210736 ] - ], - "ignore-failure" : false - } - } -} + "expected-results": { + "3x3bitmaprect_565.png": { + "allowed-digests": null, + "ignore-failure": false + }, + "3x3bitmaprect_8888.png": { + "allowed-digests": null, + "ignore-failure": false + }, + "aaclip_gpu.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 11899819492385205974 + ] + ], + "ignore-failure": false + }, + "aarectmodes_565.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 14760033689012826769 + ] + ], + "ignore-failure": false + }, + "imageblur_565.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 3359963596899141322 + ] + ], + "ignore-failure": false + }, + "imageblur_8888.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 4217923806027861152 + ] + ], + "ignore-failure": false + }, + "shadertext3_8888.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 3713708307125704716 + ] + ], + "ignore-failure": false + }, + "xfermodeimagefilter_pdf.png": { + "allowed-digests": null, + "ignore-failure": false + }, + "xfermodes_pdf.png": { + "allowed-digests": [ + [ + "bitmap-64bitMD5", + 9151974350149210736 + ] + ], + "ignore-failure": false + } + } +}
\ No newline at end of file diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/return_value b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/return_value index d00491fd7e..573541ac97 100644 --- a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/return_value +++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/return_value @@ -1 +1 @@ -1 +0 diff --git a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/stdout b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/stdout index 43c078e65c..e69de29bb2 100644 --- a/tools/tests/rebaseline/output/using-json1-expectations/output-expected/stdout +++ b/tools/tests/rebaseline/output/using-json1-expectations/output-expected/stdout @@ -1 +0,0 @@ -ERROR: JsonRebaseliner is not implemented yet. diff --git a/tools/tests/rebaseline/output/using-json1/output-expected/command_line b/tools/tests/rebaseline/output/using-json1/output-expected/command_line index 14624e14db..127846c872 100644 --- a/tools/tests/rebaseline/output/using-json1/output-expected/command_line +++ b/tools/tests/rebaseline/output/using-json1/output-expected/command_line @@ -1 +1 @@ -python tools/rebaseline.py --dry-run --json-base-url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float +python tools/rebaseline.py --dry-run --actuals-base-url file:tools/tests/rebaseline/input/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float diff --git a/tools/tests/run.sh b/tools/tests/run.sh index 1a3f40b2a6..bbcfeb9a52 100755 --- a/tools/tests/run.sh +++ b/tools/tests/run.sh @@ -267,13 +267,13 @@ REBASELINE_INPUT=tools/tests/rebaseline/input REBASELINE_OUTPUT=tools/tests/rebaseline/output # These test the old image-file expectations. -rebaseline_images_test "--expectations-root fake/expectations/path --json-base-url file:$REBASELINE_INPUT/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/subset" -rebaseline_images_test "--json-base-url file:nonexistent-path --tests test1 test2" "$REBASELINE_OUTPUT/all" -rebaseline_images_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1" -rebaseline_images_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --add-new" "$REBASELINE_OUTPUT/using-json1-add-new" +rebaseline_images_test "--expectations-root fake/expectations/path --actuals-base-url file:$REBASELINE_INPUT/json1 --tests test1 test2 --configs 565 8888 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/subset" +rebaseline_images_test "--actuals-base-url file:nonexistent-path --tests test1 test2" "$REBASELINE_OUTPUT/all" +rebaseline_images_test "--actuals-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1" +rebaseline_images_test "--actuals-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --add-new" "$REBASELINE_OUTPUT/using-json1-add-new" # These test the new JSON-format expectations. -rebaseline_test "$REBASELINE_INPUT/json1" "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1-expectations" +rebaseline_test "$REBASELINE_INPUT/json1" "--actuals-base-url $REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float" "$REBASELINE_OUTPUT/using-json1-expectations" # # Test jsondiff.py ... |