From f7928b4f33b5e899b4ac543d8d850523cbd1d6da Mon Sep 17 00:00:00 2001 From: Eric Boren Date: Fri, 28 Jul 2017 07:35:28 -0400 Subject: Add a bot to verify that generated SKSL files have not been hand-edited Bug: skia:6893 Change-Id: I54480ddd1ad1989661393500044f9164a99ce7b0 Reviewed-on: https://skia-review.googlesource.com/27700 Reviewed-by: Ravi Mistry Commit-Queue: Eric Boren --- infra/bots/gen_tasks.go | 29 ++- infra/bots/jobs.json | 1 + .../Housekeeper-PerCommit-CheckGeneratedFiles.json | 223 ++++++++++++++++++ infra/bots/recipe_modules/flavor/examples/full.py | 1 + infra/bots/recipe_modules/flavor/gn_flavor.py | 14 +- infra/bots/recipe_modules/vars/api.py | 3 +- .../Housekeeper-PerCommit-CheckGeneratedFiles.json | 253 +++++++++++++++++++++ infra/bots/recipes/check_generated_files.py | 86 +++++++ infra/bots/tasks.json | 29 +++ 9 files changed, 635 insertions(+), 4 deletions(-) create mode 100644 infra/bots/recipe_modules/flavor/examples/full.expected/Housekeeper-PerCommit-CheckGeneratedFiles.json create mode 100644 infra/bots/recipes/check_generated_files.expected/Housekeeper-PerCommit-CheckGeneratedFiles.json create mode 100644 infra/bots/recipes/check_generated_files.py (limited to 'infra') diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go index 1d84615bee..7d03526dae 100644 --- a/infra/bots/gen_tasks.go +++ b/infra/bots/gen_tasks.go @@ -588,6 +588,29 @@ func ctSKPs(b *specs.TasksCfgBuilder, name string) string { return name } +// checkGeneratedFiles verifies that no generated SKSL files have been edited +// by hand. +func checkGeneratedFiles(b *specs.TasksCfgBuilder, name string) string { + b.MustAddTask(name, &specs.TaskSpec{ + CipdPackages: []*specs.CipdPackage{}, + Dimensions: linuxGceDimensions(), + ExtraArgs: []string{ + "--workdir", "../../..", "check_generated_files", + fmt.Sprintf("repository=%s", specs.PLACEHOLDER_REPO), + fmt.Sprintf("buildername=%s", name), + fmt.Sprintf("swarm_out_dir=%s", specs.PLACEHOLDER_ISOLATED_OUTDIR), + fmt.Sprintf("revision=%s", specs.PLACEHOLDER_REVISION), + fmt.Sprintf("patch_repo=%s", specs.PLACEHOLDER_PATCH_REPO), + fmt.Sprintf("patch_storage=%s", specs.PLACEHOLDER_PATCH_STORAGE), + fmt.Sprintf("patch_issue=%s", specs.PLACEHOLDER_ISSUE), + fmt.Sprintf("patch_set=%s", specs.PLACEHOLDER_PATCHSET), + }, + Isolate: relpath("compile_skia.isolate"), + Priority: 0.8, + }) + return name +} + // housekeeper generates a Housekeeper task. Returns the name of the last task // in the generated chain of tasks, which the Job should add as a dependency. func housekeeper(b *specs.TasksCfgBuilder, name, compileTaskName string) string { @@ -868,6 +891,7 @@ func process(b *specs.TasksCfgBuilder, name string) { if parts["role"] != "Build" && name != "Housekeeper-PerCommit-BundleRecipes" && name != "Housekeeper-PerCommit-InfraTests" && + name != "Housekeeper-PerCommit-CheckGeneratedFiles" && !strings.Contains(name, "RecreateSKPs") && !strings.Contains(name, "UpdateMetaConfig") && !strings.Contains(name, "-CT_") && @@ -875,10 +899,13 @@ func process(b *specs.TasksCfgBuilder, name string) { compile(b, compileTaskName, compileTaskParts) } - // Housekeeper. + // Housekeepers. if name == "Housekeeper-PerCommit" { deps = append(deps, housekeeper(b, name, compileTaskName)) } + if name == "Housekeeper-PerCommit-CheckGeneratedFiles" { + deps = append(deps, checkGeneratedFiles(b, name)) + } // Common assets needed by the remaining bots. diff --git a/infra/bots/jobs.json b/infra/bots/jobs.json index f41367ef5b..9501fc3638 100644 --- a/infra/bots/jobs.json +++ b/infra/bots/jobs.json @@ -77,6 +77,7 @@ "Housekeeper-Nightly-UpdateMetaConfig", "Housekeeper-PerCommit", "Housekeeper-PerCommit-BundleRecipes", + "Housekeeper-PerCommit-CheckGeneratedFiles", "Housekeeper-PerCommit-InfraTests", "Housekeeper-PerCommit-IsolateSKP", "Housekeeper-PerCommit-IsolateSVG", diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/Housekeeper-PerCommit-CheckGeneratedFiles.json b/infra/bots/recipe_modules/flavor/examples/full.expected/Housekeeper-PerCommit-CheckGeneratedFiles.json new file mode 100644 index 0000000000..bbcd966d68 --- /dev/null +++ b/infra/bots/recipe_modules/flavor/examples/full.expected/Housekeeper-PerCommit-CheckGeneratedFiles.json @@ -0,0 +1,223 @@ +[ + { + "cmd": [ + "python", + "-u", + "[CUSTOM_/_B_WORK]/skia/bin/fetch-gn" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "fetch-gn" + }, + { + "cmd": [ + "python", + "-u", + "[CUSTOM_/_B_WORK]/skia/bin/fetch-clang-format" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "fetch-clang-format" + }, + { + "cmd": [ + "[CUSTOM_/_B_WORK]/skia/bin/gn", + "gen", + "[START_DIR]/out/Release", + "--args=is_debug=false skia_compile_processors=true" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "[CUSTOM_/_B_WORK]/skia/bin::RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "gn gen" + }, + { + "cmd": [ + "ninja", + "-C", + "[START_DIR]/out/Release" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "[CUSTOM_/_B_WORK]/skia/bin::RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "ninja" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "results_dir" + ], + "infra_step": true, + "name": "rmtree results_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "results_dir" + ], + "infra_step": true, + "name": "makedirs results_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "device_results_dir" + ], + "infra_step": true, + "name": "rmtree device_results_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "device_results_dir" + ], + "infra_step": true, + "name": "makedirs device_results_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[CUSTOM_/_B_WORK]/skia/infra/bots/assets/skp/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get downloaded SKP VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "42", + "[START_DIR]/tmp/SKP_VERSION" + ], + "infra_step": true, + "name": "write SKP_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[CUSTOM_/_B_WORK]/skia/infra/bots/assets/skimage/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get downloaded skimage VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "42", + "[START_DIR]/tmp/SK_IMAGE_VERSION" + ], + "infra_step": true, + "name": "write SK_IMAGE_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[CUSTOM_/_B_WORK]/skia/infra/bots/assets/svg/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get downloaded SVG VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "42", + "[START_DIR]/tmp/SVG_VERSION" + ], + "infra_step": true, + "name": "write SVG_VERSION" + }, + { + "cmd": [ + "[START_DIR]/out/Release/dm", + "--some-flag" + ], + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "dm" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipe_modules/flavor/examples/full.py b/infra/bots/recipe_modules/flavor/examples/full.py index f7d0ae5587..26775be4b8 100644 --- a/infra/bots/recipe_modules/flavor/examples/full.py +++ b/infra/bots/recipe_modules/flavor/examples/full.py @@ -72,6 +72,7 @@ TEST_BUILDERS = [ 'Build-Win-MSVC-x86_64-Debug-NoGPU', 'Build-Win-MSVC-x86_64-Release-Exceptions', 'Build-Win-MSVC-x86_64-Release-Vulkan', + 'Housekeeper-PerCommit-CheckGeneratedFiles', 'Perf-Android-Clang-NexusPlayer-GPU-PowerVR-x86-Debug-Android', 'Perf-ChromeOS-Clang-Chromebook_513C24_K01-GPU-MaliT860-arm-Release', 'Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release', diff --git a/infra/bots/recipe_modules/flavor/gn_flavor.py b/infra/bots/recipe_modules/flavor/gn_flavor.py index f02f2be962..3b81255168 100644 --- a/infra/bots/recipe_modules/flavor/gn_flavor.py +++ b/infra/bots/recipe_modules/flavor/gn_flavor.py @@ -105,6 +105,8 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): args['skia_vulkan_sdk'] = '"%s"' % linux_vulkan_sdk if 'Win' in os: args['skia_vulkan_sdk'] = '"%s"' % win_vulkan_sdk + if 'CheckGeneratedFiles' in extra_config: + args['skia_compile_processors'] = 'true' for (k,v) in { 'cc': cc, @@ -129,8 +131,16 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): with self.m.context(cwd=self.m.vars.skia_dir): self._py('fetch-gn', self.m.vars.skia_dir.join('bin', 'fetch-gn')) - self._run('gn gen', [gn, 'gen', self.out_dir, '--args=' + gn_args]) - self._run('ninja', [ninja, '-C', self.out_dir]) + env = {} + if 'CheckGeneratedFiles' in extra_config: + env['PATH'] = '%s:%%(PATH)s' % self.m.vars.skia_dir.join('bin') + self._py( + 'fetch-clang-format', + self.m.vars.skia_dir.join('bin', 'fetch-clang-format')) + + with self.m.env(env): + self._run('gn gen', [gn, 'gen', self.out_dir, '--args=' + gn_args]) + self._run('ninja', [ninja, '-C', self.out_dir]) def copy_extra_build_products(self, swarming_out_dir): configuration = self.m.vars.builder_cfg.get('configuration', '') diff --git a/infra/bots/recipe_modules/vars/api.py b/infra/bots/recipe_modules/vars/api.py index da11651802..b5f39834f4 100644 --- a/infra/bots/recipe_modules/vars/api.py +++ b/infra/bots/recipe_modules/vars/api.py @@ -44,7 +44,8 @@ class SkiaVarsApi(recipe_api.RecipeApi): '-CT_' in self.builder_name or 'Presubmit' in self.builder_name or 'InfraTests' in self.builder_name or - self.builder_name == "Housekeeper-PerCommit") + self.builder_name == "Housekeeper-PerCommit" or + 'CheckGeneratedFiles' in self.builder_name) if self.persistent_checkout: if 'Win' in self.builder_name: self.checkout_root = self.make_path('C:\\', 'b', 'work') diff --git a/infra/bots/recipes/check_generated_files.expected/Housekeeper-PerCommit-CheckGeneratedFiles.json b/infra/bots/recipes/check_generated_files.expected/Housekeeper-PerCommit-CheckGeneratedFiles.json new file mode 100644 index 0000000000..afd93821a0 --- /dev/null +++ b/infra/bots/recipes/check_generated_files.expected/Housekeeper-PerCommit-CheckGeneratedFiles.json @@ -0,0 +1,253 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[CUSTOM_/_B_WORK]" + ], + "infra_step": true, + "name": "makedirs checkout_path" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "remove", + "[CUSTOM_/_B_WORK]/.gclient_entries" + ], + "infra_step": true, + "name": "remove [CUSTOM_/_B_WORK]/.gclient_entries" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec", + "cache_dir = '[CUSTOM_/_B_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]", + "--patch_root", + "skia", + "--revision_mapping_file", + "{\"got_revision\": \"skia\"}", + "--git-cache-dir", + "[CUSTOM_/_B_CACHE]", + "--cleanup-dir", + "[CLEANUP]/bot_update", + "--output_json", + "/path/to/tmp/json", + "--revision", + "skia@abc123" + ], + "cwd": "[CUSTOM_/_B_WORK]", + "env": { + "GIT_HTTP_LOW_SPEED_LIMIT": "1000", + "GIT_HTTP_LOW_SPEED_TIME": "300", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]" + }, + "infra_step": true, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/tmp" + ], + "infra_step": true, + "name": "makedirs tmp_dir" + }, + { + "cmd": [ + "git", + "diff", + "--no-ext-diff" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "git diff #1", + "stdout": "/path/to/tmp/" + }, + { + "cmd": [ + "python", + "-u", + "import os\nimport subprocess\n\nfor r, d, files in os.walk('[CUSTOM_/_B_WORK]/skia'):\n for f in files:\n if f.endswith('.fp'):\n path = os.path.join(r, f)\n print 'touch %s' % path\n subprocess.check_call(['touch', path])\n" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "touch fp files", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import subprocess@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@for r, d, files in os.walk('[CUSTOM_/_B_WORK]/skia'):@@@", + "@@@STEP_LOG_LINE@python.inline@ for f in files:@@@", + "@@@STEP_LOG_LINE@python.inline@ if f.endswith('.fp'):@@@", + "@@@STEP_LOG_LINE@python.inline@ path = os.path.join(r, f)@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'touch %s' % path@@@", + "@@@STEP_LOG_LINE@python.inline@ subprocess.check_call(['touch', path])@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "[CUSTOM_/_B_WORK]/skia/bin/fetch-gn" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "fetch-gn" + }, + { + "cmd": [ + "python", + "-u", + "[CUSTOM_/_B_WORK]/skia/bin/fetch-clang-format" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "infra_step": true, + "name": "fetch-clang-format" + }, + { + "cmd": [ + "[CUSTOM_/_B_WORK]/skia/bin/gn", + "gen", + "[START_DIR]/out/Release", + "--args=is_debug=false skia_compile_processors=true" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "[CUSTOM_/_B_WORK]/skia/bin::RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "gn gen" + }, + { + "cmd": [ + "ninja", + "-C", + "[START_DIR]/out/Release" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": "[CUSTOM_/_B_WORK]/skia/bin::RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "ninja" + }, + { + "cmd": [ + "git", + "diff", + "--no-ext-diff" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "git diff #2", + "stdout": "/path/to/tmp/" + }, + { + "cmd": [ + "python", + "-u", + "\ndiff1 = ''''''\n\ndiff2 = ''''''\n\nif diff1 != diff2:\n print 'Generated files have been edited!'\n exit(1)\n" + ], + "cwd": "[CUSTOM_/_B_WORK]/skia", + "env": { + "BUILDTYPE": "Release", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[START_DIR]/out" + }, + "name": "compare diffs", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@diff1 = ''''''@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@diff2 = ''''''@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@if diff1 != diff2:@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Generated files have been edited!'@@@", + "@@@STEP_LOG_LINE@python.inline@ exit(1)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipes/check_generated_files.py b/infra/bots/recipes/check_generated_files.py new file mode 100644 index 0000000000..fc4f79b02a --- /dev/null +++ b/infra/bots/recipes/check_generated_files.py @@ -0,0 +1,86 @@ +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +# Recipe for the Skia PerCommit Housekeeper. + +DEPS = [ + 'recipe_engine/context', + 'recipe_engine/path', + 'recipe_engine/properties', + 'recipe_engine/python', + 'recipe_engine/raw_io', + 'recipe_engine/step', + 'core', + 'flavor', + 'run', + 'vars', +] + + +def RunSteps(api): + # Checkout, compile, etc. + api.core.setup() + + cwd = api.path['checkout'] + + with api.context(cwd=cwd): + # Get a baseline diff. This should be empty, but we want to be flexible for + # cases where we have local diffs on purpose. + diff1 = api.run( + api.step, + 'git diff #1', + cmd=['git', 'diff', '--no-ext-diff'], + stdout=api.m.raw_io.output()).stdout + + # Touch all .fp files so that the generated files are rebuilt. + api.run( + api.python.inline, + 'touch fp files', + program="""import os +import subprocess + +for r, d, files in os.walk('%s'): + for f in files: + if f.endswith('.fp'): + path = os.path.join(r, f) + print 'touch %%s' %% path + subprocess.check_call(['touch', path]) +""" % cwd) + + # Regenerate the SKSL files. + api.flavor.compile('compile_processors') + + # Get a second diff. If this doesn't match the first, then there have been + # modifications to the generated files. + diff2 = api.run( + api.step, + 'git diff #2', + cmd=['git', 'diff', '--no-ext-diff'], + stdout=api.m.raw_io.output()).stdout + + api.run( + api.python.inline, + 'compare diffs', + program=""" +diff1 = '''%s''' + +diff2 = '''%s''' + +if diff1 != diff2: + print 'Generated files have been edited!' + exit(1) +""" % (diff1, diff2)) + + +def GenTests(api): + yield ( + api.test('Housekeeper-PerCommit-CheckGeneratedFiles') + + api.properties(buildername='Housekeeper-PerCommit-CheckGeneratedFiles', + repository='https://skia.googlesource.com/skia.git', + revision='abc123', + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + + api.path.exists(api.path['start_dir']) + ) diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json index 4643563e27..34cdea7c77 100644 --- a/infra/bots/tasks.json +++ b/infra/bots/tasks.json @@ -470,6 +470,12 @@ "Housekeeper-PerCommit-BundleRecipes" ] }, + "Housekeeper-PerCommit-CheckGeneratedFiles": { + "priority": 0.8, + "tasks": [ + "Housekeeper-PerCommit-CheckGeneratedFiles" + ] + }, "Housekeeper-PerCommit-InfraTests": { "priority": 0.8, "tasks": [ @@ -4504,6 +4510,29 @@ "isolate": "bundle_recipes.isolate", "priority": 0.7 }, + "Housekeeper-PerCommit-CheckGeneratedFiles": { + "dimensions": [ + "cpu:x86-64-avx2", + "gpu:none", + "os:Debian-9.0", + "pool:Skia" + ], + "extra_args": [ + "--workdir", + "../../..", + "check_generated_files", + "repository=<(REPO)", + "buildername=Housekeeper-PerCommit-CheckGeneratedFiles", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "revision=<(REVISION)", + "patch_repo=<(PATCH_REPO)", + "patch_storage=<(PATCH_STORAGE)", + "patch_issue=<(ISSUE)", + "patch_set=<(PATCHSET)" + ], + "isolate": "compile_skia.isolate", + "priority": 0.8 + }, "Housekeeper-PerCommit-InfraTests": { "cipd_packages": [ { -- cgit v1.2.3