diff options
author | 2014-01-03 16:04:11 +0000 | |
---|---|---|
committer | 2014-01-03 16:04:11 +0000 | |
commit | 0bf919c89a5cd8d1779cdbd5d17536c2fbf534c8 (patch) | |
tree | c3b4cf408291039b44e7b4fb5e5c0ea77a0bf04b /gm | |
parent | dcea5300dc028b5ea210d778b9e34a7504fee6d2 (diff) |
make results_test.py work better within python unittest framework
BUG=
R=rmistry@google.com
Author: epoger@google.com
Review URL: https://codereview.chromium.org/122443006
git-svn-id: http://skia.googlecode.com/svn/trunk@12882 2bbb7eff-a529-9590-31e7-b0007b416f81
Diffstat (limited to 'gm')
-rwxr-xr-x | gm/rebaseline_server/results_test.py | 57 | ||||
-rw-r--r-- | gm/rebaseline_server/tests/outputs/expected/results_test.ResultsTest.test_gm/gm.json (renamed from gm/rebaseline_server/tests/outputs/expected/gm.json) | 0 |
2 files changed, 41 insertions, 16 deletions
diff --git a/gm/rebaseline_server/results_test.py b/gm/rebaseline_server/results_test.py index 94e9ca2de9..e5f969c5ea 100755 --- a/gm/rebaseline_server/results_test.py +++ b/gm/rebaseline_server/results_test.py @@ -13,12 +13,12 @@ on the housekeeper bot, but first make sure it works properly after having been checked out (from both git and svn) TODO(epoger): Create a command to update the expected results (in -OUTPUT_DIR_EXPECTATIONS) when appropriate. For now, you should: -1. examine the results in OUTPUT_DIR and make sure they are ok -2. rm -rf OUTPUT_DIR_EXPECTATIONS -3. mv OUTPUT_DIR OUTPUT_DIR_EXPECTATIONS +OUTPUT_DIR_EXPECTED) when appropriate. For now, you should: +1. examine the results in OUTPUT_DIR_ACTUAL and make sure they are ok +2. rm -rf OUTPUT_DIR_EXPECTED +3. mv OUTPUT_DIR_ACTUAL OUTPUT_DIR_EXPECTED Although, if you're using an SVN checkout, this will blow away .svn directories -within OUTPUT_DIR_EXPECTATIONS, which wouldn't be good... +within OUTPUT_DIR_EXPECTED, which wouldn't be good... """ @@ -35,26 +35,55 @@ import gm_json # must import results first, so that gm_json will be in sys.path PARENT_DIR = os.path.dirname(os.path.realpath(__file__)) INPUT_DIR = os.path.join(PARENT_DIR, 'tests', 'inputs') -OUTPUT_DIR = os.path.join(PARENT_DIR, 'tests', 'outputs', 'actual') -OUTPUT_DIR_EXPECTATIONS = os.path.join( - PARENT_DIR, 'tests', 'outputs', 'expected') +OUTPUT_DIR_ACTUAL = os.path.join(PARENT_DIR, 'tests', 'outputs', 'actual') +OUTPUT_DIR_EXPECTED = os.path.join(PARENT_DIR, 'tests', 'outputs', 'expected') + class ResultsTest(unittest.TestCase): def setUp(self): - self._tempdir = tempfile.mkdtemp() + self._temp_dir = tempfile.mkdtemp() + self._output_dir_actual = os.path.join(OUTPUT_DIR_ACTUAL, self.id()) + self._output_dir_expected = os.path.join(OUTPUT_DIR_EXPECTED, self.id()) + create_empty_dir(self._output_dir_actual) def tearDown(self): - shutil.rmtree(self._tempdir) + shutil.rmtree(self._temp_dir) + different_files = find_different_files(self._output_dir_actual, + self._output_dir_expected) + # Maybe we should move this assert elsewhere? It's unusual to see an + # assert within tearDown(), but my thinking was: + # 1. Every test case will have some collection of output files that need to + # be validated. + # 2. So put that validation within tearDown(), which will be called after + # every test case! + # + # I have confirmed that the test really does fail if this assert is + # triggered. + # + # Ravi notes: if somebody later comes along and adds cleanup code below the + # assert, then if tests fail, the artifacts will not be cleaned up. + assert (not different_files), \ + ('found differing files between actual dir %s and expected dir %s: %s' % + (self._output_dir_actual, self._output_dir_expected, different_files)) def test_gm(self): """Process results of a GM run with the Results object.""" results_obj = results.Results( actuals_root=os.path.join(INPUT_DIR, 'gm-actuals'), expected_root=os.path.join(INPUT_DIR, 'gm-expectations'), - generated_images_root=self._tempdir) + generated_images_root=self._temp_dir) gm_json.WriteToFile(results_obj.get_results_of_type(results.RESULTS_ALL), - os.path.join(OUTPUT_DIR, 'gm.json')) + os.path.join(self._output_dir_actual, 'gm.json')) + + +def create_empty_dir(path): + """Create an empty directory at the given path.""" + if os.path.isdir(path): + shutil.rmtree(path) + elif os.path.lexists(path): + os.remove(path) + os.makedirs(path) def find_different_files(dir1, dir2, ignore_subtree_names=None): @@ -86,12 +115,8 @@ def find_different_files(dir1, dir2, ignore_subtree_names=None): def main(): - if not os.path.isdir(OUTPUT_DIR): - os.makedirs(OUTPUT_DIR) suite = unittest.TestLoader().loadTestsFromTestCase(ResultsTest) unittest.TextTestRunner(verbosity=2).run(suite) - different_files = find_different_files(OUTPUT_DIR, OUTPUT_DIR_EXPECTATIONS) - assert (not different_files), 'found differing files: %s' % different_files if __name__ == '__main__': diff --git a/gm/rebaseline_server/tests/outputs/expected/gm.json b/gm/rebaseline_server/tests/outputs/expected/results_test.ResultsTest.test_gm/gm.json index 89ad21b241..89ad21b241 100644 --- a/gm/rebaseline_server/tests/outputs/expected/gm.json +++ b/gm/rebaseline_server/tests/outputs/expected/results_test.ResultsTest.test_gm/gm.json |