From 3f0451772109959fcb79bacf2c9a03e0eb39ff27 Mon Sep 17 00:00:00 2001 From: "commit-bot@chromium.org" Date: Thu, 15 May 2014 15:10:48 +0000 Subject: render_pictures: add --mismatchPath flag When set, it will only write out images that don't match expectations. BUG=skia:1942 R=rmistry@google.com Author: epoger@google.com Review URL: https://codereview.chromium.org/283123002 git-svn-id: http://skia.googlecode.com/svn/trunk@14748 2bbb7eff-a529-9590-31e7-b0007b416f81 --- tools/tests/render_pictures_test.py | 106 ++++++++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 30 deletions(-) (limited to 'tools/tests/render_pictures_test.py') diff --git a/tools/tests/render_pictures_test.py b/tools/tests/render_pictures_test.py index 4b11e56ae9..5ab9d673bb 100755 --- a/tools/tests/render_pictures_test.py +++ b/tools/tests/render_pictures_test.py @@ -156,12 +156,13 @@ class RenderPicturesTest(base_unittest.TestCase): self.maxDiff = MAX_DIFF_LENGTH self._expectations_dir = tempfile.mkdtemp() self._input_skp_dir = tempfile.mkdtemp() - self._temp_dir = tempfile.mkdtemp() + # All output of render_pictures binary will go into this directory. + self._output_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self._expectations_dir) shutil.rmtree(self._input_skp_dir) - shutil.rmtree(self._temp_dir) + shutil.rmtree(self._output_dir) def test_tiled_whole_image(self): """Run render_pictures with tiles and --writeWholeImage flag. @@ -169,6 +170,7 @@ class RenderPicturesTest(base_unittest.TestCase): TODO(epoger): This test generates undesired results! The JSON summary includes both whole-image and tiled-images (as it should), but only whole-images are written out to disk. See http://skbug.com/2463 + Once I fix that, I should add a similar test that exercises mismatchPath. TODO(epoger): I noticed that when this is run without --writePath being specified, this test writes red_skp.png and green_skp.png to the current @@ -176,7 +178,9 @@ class RenderPicturesTest(base_unittest.TestCase): probably shouldn't write out red_skp.png and green_skp.png at all! See http://skbug.com/2464 """ - output_json_path = os.path.join(self._temp_dir, 'actuals.json') + output_json_path = os.path.join(self._output_dir, 'actuals.json') + write_path_dir = self.create_empty_dir( + path=os.path.join(self._output_dir, 'writePath')) self._generate_skps() expectations_path = self._create_expectations() self._run_render_pictures([ @@ -185,7 +189,7 @@ class RenderPicturesTest(base_unittest.TestCase): '--mode', 'tile', '256', '256', '--readJsonSummaryPath', expectations_path, '--writeJsonSummaryPath', output_json_path, - '--writePath', self._temp_dir, + '--writePath', write_path_dir, '--writeWholeImage']) expected_summary_dict = { "header" : EXPECTED_HEADER_CONTENTS, @@ -202,12 +206,14 @@ class RenderPicturesTest(base_unittest.TestCase): } self._assert_json_contents(output_json_path, expected_summary_dict) self._assert_directory_contents( - self._temp_dir, ['red_skp.png', 'green_skp.png', 'actuals.json']) + write_path_dir, ['red_skp.png', 'green_skp.png']) def test_missing_tile_and_whole_image(self): """test_tiled_whole_image, but missing expectations for some images. """ - output_json_path = os.path.join(self._temp_dir, 'actuals.json') + output_json_path = os.path.join(self._output_dir, 'actuals.json') + write_path_dir = self.create_empty_dir( + path=os.path.join(self._output_dir, 'writePath')) self._generate_skps() expectations_path = self._create_expectations(missing_some_images=True) self._run_render_pictures([ @@ -216,7 +222,7 @@ class RenderPicturesTest(base_unittest.TestCase): '--mode', 'tile', '256', '256', '--readJsonSummaryPath', expectations_path, '--writeJsonSummaryPath', output_json_path, - '--writePath', self._temp_dir, + '--writePath', write_path_dir, '--writeWholeImage']) modified_red_tiles = copy.deepcopy(RED_TILES) modified_red_tiles[5]['comparisonResult'] = 'no-comparison' @@ -238,13 +244,15 @@ class RenderPicturesTest(base_unittest.TestCase): def test_untiled(self): """Run without tiles.""" - output_json_path = os.path.join(self._temp_dir, 'actuals.json') + output_json_path = os.path.join(self._output_dir, 'actuals.json') + write_path_dir = self.create_empty_dir( + path=os.path.join(self._output_dir, 'writePath')) self._generate_skps() expectations_path = self._create_expectations() self._run_render_pictures([ '-r', self._input_skp_dir, '--readJsonSummaryPath', expectations_path, - '--writePath', self._temp_dir, + '--writePath', write_path_dir, '--writeJsonSummaryPath', output_json_path]) expected_summary_dict = { "header" : EXPECTED_HEADER_CONTENTS, @@ -259,15 +267,17 @@ class RenderPicturesTest(base_unittest.TestCase): } self._assert_json_contents(output_json_path, expected_summary_dict) self._assert_directory_contents( - self._temp_dir, ['red_skp.png', 'green_skp.png', 'actuals.json']) + write_path_dir, ['red_skp.png', 'green_skp.png']) def test_untiled_writeChecksumBasedFilenames(self): """Same as test_untiled, but with --writeChecksumBasedFilenames.""" - output_json_path = os.path.join(self._temp_dir, 'actuals.json') + output_json_path = os.path.join(self._output_dir, 'actuals.json') + write_path_dir = self.create_empty_dir( + path=os.path.join(self._output_dir, 'writePath')) self._generate_skps() self._run_render_pictures(['-r', self._input_skp_dir, '--writeChecksumBasedFilenames', - '--writePath', self._temp_dir, + '--writePath', write_path_dir, '--writeJsonSummaryPath', output_json_path]) expected_summary_dict = { "header" : EXPECTED_HEADER_CONTENTS, @@ -293,25 +303,26 @@ class RenderPicturesTest(base_unittest.TestCase): } } self._assert_json_contents(output_json_path, expected_summary_dict) - self._assert_directory_contents(self._temp_dir, [ - 'red_skp', 'green_skp', 'actuals.json']) + self._assert_directory_contents(write_path_dir, ['red_skp', 'green_skp']) self._assert_directory_contents( - os.path.join(self._temp_dir, 'red_skp'), + os.path.join(write_path_dir, 'red_skp'), ['bitmap-64bitMD5_11092453015575919668.png']) self._assert_directory_contents( - os.path.join(self._temp_dir, 'green_skp'), + os.path.join(write_path_dir, 'green_skp'), ['bitmap-64bitMD5_8891695120562235492.png']) def test_untiled_validate(self): """Same as test_untiled, but with --validate.""" - output_json_path = os.path.join(self._temp_dir, 'actuals.json') + output_json_path = os.path.join(self._output_dir, 'actuals.json') + write_path_dir = self.create_empty_dir( + path=os.path.join(self._output_dir, 'writePath')) self._generate_skps() expectations_path = self._create_expectations() self._run_render_pictures([ '-r', self._input_skp_dir, '--readJsonSummaryPath', expectations_path, '--validate', - '--writePath', self._temp_dir, + '--writePath', write_path_dir, '--writeJsonSummaryPath', output_json_path]) expected_summary_dict = { "header" : EXPECTED_HEADER_CONTENTS, @@ -326,11 +337,11 @@ class RenderPicturesTest(base_unittest.TestCase): } self._assert_json_contents(output_json_path, expected_summary_dict) self._assert_directory_contents( - self._temp_dir, ['red_skp.png', 'green_skp.png', 'actuals.json']) + write_path_dir, ['red_skp.png', 'green_skp.png']) def test_untiled_without_writePath(self): """Same as test_untiled, but without --writePath.""" - output_json_path = os.path.join(self._temp_dir, 'actuals.json') + output_json_path = os.path.join(self._output_dir, 'actuals.json') self._generate_skps() expectations_path = self._create_expectations() self._run_render_pictures([ @@ -352,7 +363,9 @@ class RenderPicturesTest(base_unittest.TestCase): def test_tiled(self): """Generate individual tiles.""" - output_json_path = os.path.join(self._temp_dir, 'actuals.json') + output_json_path = os.path.join(self._output_dir, 'actuals.json') + write_path_dir = self.create_empty_dir( + path=os.path.join(self._output_dir, 'writePath')) self._generate_skps() expectations_path = self._create_expectations() self._run_render_pictures([ @@ -360,7 +373,7 @@ class RenderPicturesTest(base_unittest.TestCase): '--bbh', 'grid', '256', '256', '--mode', 'tile', '256', '256', '--readJsonSummaryPath', expectations_path, - '--writePath', self._temp_dir, + '--writePath', write_path_dir, '--writeJsonSummaryPath', output_json_path]) expected_summary_dict = { "header" : EXPECTED_HEADER_CONTENTS, @@ -375,22 +388,56 @@ class RenderPicturesTest(base_unittest.TestCase): } self._assert_json_contents(output_json_path, expected_summary_dict) self._assert_directory_contents( - self._temp_dir, + write_path_dir, ['red_skp-tile0.png', 'red_skp-tile1.png', 'red_skp-tile2.png', 'red_skp-tile3.png', 'red_skp-tile4.png', 'red_skp-tile5.png', 'green_skp-tile0.png', 'green_skp-tile1.png', 'green_skp-tile2.png', 'green_skp-tile3.png', 'green_skp-tile4.png', 'green_skp-tile5.png', - 'actuals.json']) + ]) + + def test_tiled_mismatches(self): + """Same as test_tiled, but only write out mismatching images.""" + output_json_path = os.path.join(self._output_dir, 'actuals.json') + mismatch_path_dir = self.create_empty_dir( + path=os.path.join(self._output_dir, 'mismatchPath')) + self._generate_skps() + expectations_path = self._create_expectations() + self._run_render_pictures([ + '-r', self._input_skp_dir, + '--bbh', 'grid', '256', '256', + '--mode', 'tile', '256', '256', + '--readJsonSummaryPath', expectations_path, + '--mismatchPath', mismatch_path_dir, + '--writeJsonSummaryPath', output_json_path]) + expected_summary_dict = { + "header" : EXPECTED_HEADER_CONTENTS, + "actual-results" : { + "red.skp": { + "tiled-images": RED_TILES, + }, + "green.skp": { + "tiled-images": GREEN_TILES, + } + } + } + self._assert_json_contents(output_json_path, expected_summary_dict) + self._assert_directory_contents( + mismatch_path_dir, + ['red_skp-tile0.png', 'red_skp-tile1.png', 'red_skp-tile2.png', + 'red_skp-tile3.png', 'red_skp-tile4.png', 'red_skp-tile5.png', + ]) def test_tiled_writeChecksumBasedFilenames(self): """Same as test_tiled, but with --writeChecksumBasedFilenames.""" - output_json_path = os.path.join(self._temp_dir, 'actuals.json') + output_json_path = os.path.join(self._output_dir, 'actuals.json') + write_path_dir = self.create_empty_dir( + path=os.path.join(self._output_dir, 'writePath')) self._generate_skps() self._run_render_pictures(['-r', self._input_skp_dir, '--bbh', 'grid', '256', '256', '--mode', 'tile', '256', '256', '--writeChecksumBasedFilenames', - '--writePath', self._temp_dir, + '--writePath', write_path_dir, '--writeJsonSummaryPath', output_json_path]) expected_summary_dict = { "header" : EXPECTED_HEADER_CONTENTS, @@ -470,10 +517,9 @@ class RenderPicturesTest(base_unittest.TestCase): } } self._assert_json_contents(output_json_path, expected_summary_dict) - self._assert_directory_contents(self._temp_dir, [ - 'red_skp', 'green_skp', 'actuals.json']) + self._assert_directory_contents(write_path_dir, ['red_skp', 'green_skp']) self._assert_directory_contents( - os.path.join(self._temp_dir, 'red_skp'), + os.path.join(write_path_dir, 'red_skp'), ['bitmap-64bitMD5_5815827069051002745.png', 'bitmap-64bitMD5_9323613075234140270.png', 'bitmap-64bitMD5_16670399404877552232.png', @@ -481,7 +527,7 @@ class RenderPicturesTest(base_unittest.TestCase): 'bitmap-64bitMD5_7325267995523877959.png', 'bitmap-64bitMD5_2181381724594493116.png']) self._assert_directory_contents( - os.path.join(self._temp_dir, 'green_skp'), + os.path.join(write_path_dir, 'green_skp'), ['bitmap-64bitMD5_12587324416545178013.png', 'bitmap-64bitMD5_7624374914829746293.png', 'bitmap-64bitMD5_5686489729535631913.png', -- cgit v1.2.3