From afaad3dd7002feb6e69983ddc4e5148d7803baed Mon Sep 17 00:00:00 2001 From: "epoger@google.com" Date: Mon, 30 Sep 2013 15:06:25 +0000 Subject: HTTP GM results viewer: server now returns category summaries along with testData (SkipBuildbotRuns) R=borenet@google.com Review URL: https://codereview.chromium.org/25045003 git-svn-id: http://skia.googlecode.com/svn/trunk@11520 2bbb7eff-a529-9590-31e7-b0007b416f81 --- gm/rebaseline_server/results.py | 119 ++++++++++++++++++++++++++++++---------- 1 file changed, 89 insertions(+), 30 deletions(-) (limited to 'gm/rebaseline_server/results.py') diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/results.py index 0c50d26397..1336097f1f 100755 --- a/gm/rebaseline_server/results.py +++ b/gm/rebaseline_server/results.py @@ -31,6 +31,9 @@ if GM_DIRECTORY not in sys.path: import gm_json IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) +CATEGORIES_TO_SUMMARIZE = [ + 'builder', 'test', 'config', 'resultType', +] class Results(object): """ Loads actual and expected results from all builders, supplying combined @@ -44,24 +47,49 @@ class Results(object): """ self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root) self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root) - self._all_results = self._Combine() + self._all_results = Results._Combine( + actual_builder_dicts=self._actual_builder_dicts, + expected_builder_dicts=self._expected_builder_dicts) def GetAll(self): - """Return results of all tests, as a list in this form: + """Return results of all tests, as a dictionary in this form: - [ + { + "categories": # dictionary of categories listed in + # CATEGORIES_TO_SUMMARIZE, with the number of times + # each value appears within its category { - "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug", - "test": "bigmatrix", - "config": "8888", - "resultType": "failed", - "expectedHashType": "bitmap-64bitMD5", - "expectedHashDigest": "10894408024079689926", - "actualHashType": "bitmap-64bitMD5", - "actualHashDigest": "2409857384569", - }, - ... - ] + "resultType": # category name + { + "failed": 29, # category value and total number found of that value + "failure-ignored": 948, + "no-comparison": 4502, + "succeeded": 38609, + }, + "builder": + { + "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug": 1286, + "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release": 1134, + ... + }, + ... # other categories from CATEGORIES_TO_SUMMARIZE + }, # end of "categories" dictionary + + "testData": # list of test results, with a dictionary for each + [ + { + "builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug", + "test": "bigmatrix", + "config": "8888", + "resultType": "failed", + "expectedHashType": "bitmap-64bitMD5", + "expectedHashDigest": "10894408024079689926", + "actualHashType": "bitmap-64bitMD5", + "actualHashDigest": "2409857384569", + }, + ... + ], # end of "testData" list + } """ return self._all_results @@ -84,15 +112,21 @@ class Results(object): meta_dict[builder] = gm_json.LoadFromFile(fullpath) return meta_dict - def _Combine(self): - """Returns a list of all tests, across all builders, based on the - contents of self._actual_builder_dicts and self._expected_builder_dicts . - Returns the list in the same form needed for GetAllResults(). + @staticmethod + def _Combine(actual_builder_dicts, expected_builder_dicts): + """Gathers the results of all tests, across all builders (based on the + contents of actual_builder_dicts and expected_builder_dicts) + and returns it in a list in the same form needed for self.GetAll(). + + This is a static method, because once we start refreshing results + asynchronously, we need to make sure we are not corrupting the object's + member variables. """ - all_tests = [] - for builder in sorted(self._actual_builder_dicts.keys()): + test_data = [] + category_dict = {} + for builder in sorted(actual_builder_dicts.keys()): actual_results_for_this_builder = ( - self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) + actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) for result_type in sorted(actual_results_for_this_builder.keys()): results_of_this_type = actual_results_for_this_builder[result_type] if not results_of_this_type: @@ -102,7 +136,7 @@ class Results(object): try: # TODO(epoger): assumes a single allowed digest per test expected_image = ( - self._expected_builder_dicts + expected_builder_dicts [builder][gm_json.JSONKEY_EXPECTEDRESULTS] [image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS] [0]) @@ -159,13 +193,8 @@ class Results(object): else: updated_result_type = result_type - # TODO(epoger): For now, don't include succeeded results. - # There are so many of them that they make the client too slow. - if updated_result_type == gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED: - continue - (test, config) = IMAGE_FILENAME_RE.match(image_name).groups() - all_tests.append({ + results_for_this_test = { "builder": builder, "test": test, "config": config, @@ -174,5 +203,35 @@ class Results(object): "actualHashDigest": str(actual_image[1]), "expectedHashType": expected_image[0], "expectedHashDigest": str(expected_image[1]), - }) - return all_tests + } + Results._AddToCategoryDict(category_dict, results_for_this_test) + + # TODO(epoger): For now, don't include succeeded results in the raw + # data. There are so many of them that they make the client too slow. + if updated_result_type != gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED: + test_data.append(results_for_this_test) + return {"categories": category_dict, "testData": test_data} + + @staticmethod + def _AddToCategoryDict(category_dict, test_results): + """Add test_results to the category dictionary we are building + (see documentation of self.GetAll() for the format of this dictionary). + + params: + category_dict: category dict-of-dicts to add to; modify this in-place + test_results: test data with which to update category_list, in a dict: + { + "category_name": "category_value", + "category_name": "category_value", + ... + } + """ + for category in CATEGORIES_TO_SUMMARIZE: + category_value = test_results.get(category) + if not category_value: + continue # test_results did not include this category, keep going + if not category_dict.get(category): + category_dict[category] = {} + if not category_dict[category].get(category_value): + category_dict[category][category_value] = 0 + category_dict[category][category_value] += 1 -- cgit v1.2.3