From 8c172ba397087850e9c4949a42ae558ebc6fdaf4 Mon Sep 17 00:00:00 2001 From: Eric Boren Date: Thu, 19 Jul 2018 13:27:49 -0400 Subject: [infra] Add infra support for Lottie Bug: skia:8136 Change-Id: I18c4ad549c52346ebfe23d172597d5da205e5c4d Reviewed-on: https://skia-review.googlesource.com/142105 Commit-Queue: Eric Boren Reviewed-by: Ben Wagner --- infra/bots/assets/asset_utils.py | 15 +- infra/bots/assets/lottie-samples/VERSION | 1 + infra/bots/assets/lottie-samples/common.py | 26 ++ infra/bots/assets/lottie-samples/create.py | 28 ++ .../assets/lottie-samples/create_and_upload.py | 42 ++ infra/bots/assets/lottie-samples/download.py | 16 + infra/bots/assets/lottie-samples/upload.py | 16 + infra/bots/gen_tasks.go | 3 + infra/bots/recipe_modules/flavor/android.py | 1 + infra/bots/recipe_modules/flavor/api.py | 29 +- infra/bots/recipe_modules/flavor/chromebook.py | 1 + infra/bots/recipe_modules/flavor/chromecast.py | 1 + infra/bots/recipe_modules/flavor/default.py | 7 + ...ang-GCE-CPU-AVX2-x86_64-Release-All-Lottie.json | 108 +++++ infra/bots/recipe_modules/flavor/examples/full.py | 7 +- infra/bots/recipe_modules/flavor/ios.py | 1 + infra/bots/recipes/perf.py | 2 +- ...ang-GCE-CPU-AVX2-x86_64-Release-All-Lottie.json | 478 +++++++++++++++++++++ infra/bots/recipes/test.py | 35 +- 19 files changed, 794 insertions(+), 23 deletions(-) create mode 100644 infra/bots/assets/lottie-samples/VERSION create mode 100755 infra/bots/assets/lottie-samples/common.py create mode 100755 infra/bots/assets/lottie-samples/create.py create mode 100755 infra/bots/assets/lottie-samples/create_and_upload.py create mode 100755 infra/bots/assets/lottie-samples/download.py create mode 100755 infra/bots/assets/lottie-samples/upload.py create mode 100644 infra/bots/recipe_modules/flavor/examples/full.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-Lottie.json create mode 100644 infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-Lottie.json (limited to 'infra') diff --git a/infra/bots/assets/asset_utils.py b/infra/bots/assets/asset_utils.py index 59d7ca0e44..e86035efa0 100644 --- a/infra/bots/assets/asset_utils.py +++ b/infra/bots/assets/asset_utils.py @@ -73,11 +73,9 @@ class CIPDStore(object): # Enable automatic GCE authentication. For context see # https://bugs.chromium.org/p/skia/issues/detail?id=6385#c3 cipd_args.extend(['-service-account-json', ':gce']) - subprocess.check_call( - [self._cipd] - + cmd - + cipd_args - ) + return subprocess.check_output( + [self._cipd] + cmd + cipd_args, + stderr=subprocess.STDOUT) def _json_output(self, cmd): """Run the given command, return the JSON output.""" @@ -89,7 +87,12 @@ class CIPDStore(object): return parsed.get('result', []) def _search(self, pkg_name): - res = self._json_output(['search', pkg_name, '--tag', TAG_PROJECT_SKIA]) + try: + res = self._json_output(['search', pkg_name, '--tag', TAG_PROJECT_SKIA]) + except subprocess.CalledProcessError as e: + if 'no such package' in e.output: + return [] + raise return [r['instance_id'] for r in res] def _describe(self, pkg_name, instance_id): diff --git a/infra/bots/assets/lottie-samples/VERSION b/infra/bots/assets/lottie-samples/VERSION new file mode 100644 index 0000000000..c227083464 --- /dev/null +++ b/infra/bots/assets/lottie-samples/VERSION @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/infra/bots/assets/lottie-samples/common.py b/infra/bots/assets/lottie-samples/common.py new file mode 100755 index 0000000000..caa0ad899c --- /dev/null +++ b/infra/bots/assets/lottie-samples/common.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# +# Copyright 2017 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Common vars used by scripts in this directory.""" + + +import os +import sys + +FILE_DIR = os.path.dirname(os.path.abspath(__file__)) +INFRA_BOTS_DIR = os.path.realpath(os.path.join(FILE_DIR, os.pardir, os.pardir)) + +sys.path.insert(0, INFRA_BOTS_DIR) +from assets import assets + +ASSET_NAME = os.path.basename(FILE_DIR) + + +def run(cmd): + """Run a command, eg. "upload" or "download". """ + assets.main([cmd, ASSET_NAME] + sys.argv[1:]) diff --git a/infra/bots/assets/lottie-samples/create.py b/infra/bots/assets/lottie-samples/create.py new file mode 100755 index 0000000000..7f67e12694 --- /dev/null +++ b/infra/bots/assets/lottie-samples/create.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# +# Copyright 2017 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Create the asset.""" + + +import argparse + + +def create_asset(target_dir): + """Create the asset.""" + raise NotImplementedError('Implement me!') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--target_dir', '-t', required=True) + args = parser.parse_args() + create_asset(args.target_dir) + + +if __name__ == '__main__': + main() diff --git a/infra/bots/assets/lottie-samples/create_and_upload.py b/infra/bots/assets/lottie-samples/create_and_upload.py new file mode 100755 index 0000000000..de56a80fa8 --- /dev/null +++ b/infra/bots/assets/lottie-samples/create_and_upload.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# +# Copyright 2017 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Create the asset and upload it.""" + + +import argparse +import common +import os +import subprocess +import sys +import utils + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--gsutil') + args = parser.parse_args() + + with utils.tmp_dir(): + cwd = os.getcwd() + create_script = os.path.join(common.FILE_DIR, 'create.py') + upload_script = os.path.join(common.FILE_DIR, 'upload.py') + + try: + subprocess.check_call(['python', create_script, '-t', cwd]) + cmd = ['python', upload_script, '-t', cwd] + if args.gsutil: + cmd.extend(['--gsutil', args.gsutil]) + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + # Trap exceptions to avoid printing two stacktraces. + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/infra/bots/assets/lottie-samples/download.py b/infra/bots/assets/lottie-samples/download.py new file mode 100755 index 0000000000..ca999e0378 --- /dev/null +++ b/infra/bots/assets/lottie-samples/download.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# +# Copyright 2017 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Download the current version of the asset.""" + + +import common + + +if __name__ == '__main__': + common.run('download') diff --git a/infra/bots/assets/lottie-samples/upload.py b/infra/bots/assets/lottie-samples/upload.py new file mode 100755 index 0000000000..bdfbda783e --- /dev/null +++ b/infra/bots/assets/lottie-samples/upload.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# +# Copyright 2017 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Upload a new version of the asset.""" + + +import common + + +if __name__ == '__main__': + common.run('upload') diff --git a/infra/bots/gen_tasks.go b/infra/bots/gen_tasks.go index 68d2db6ea8..31bd5b8141 100644 --- a/infra/bots/gen_tasks.go +++ b/infra/bots/gen_tasks.go @@ -1003,6 +1003,9 @@ func test(b *specs.TasksCfgBuilder, name string, parts map[string]string, compil } task := kitchenTask(name, recipe, "test_skia_bundled.isolate", "", swarmDimensions(parts), extraProps, OUTPUT_TEST) task.CipdPackages = append(task.CipdPackages, pkgs...) + if strings.Contains(name, "Lottie") { + task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("lottie-samples")) + } task.Dependencies = append(task.Dependencies, compileTaskName) if strings.Contains(name, "Android_ASAN") { task.Dependencies = append(task.Dependencies, isolateCIPDAsset(b, ISOLATE_NDK_LINUX_NAME)) diff --git a/infra/bots/recipe_modules/flavor/android.py b/infra/bots/recipe_modules/flavor/android.py index 0d83c438e5..a5f2725bbe 100644 --- a/infra/bots/recipe_modules/flavor/android.py +++ b/infra/bots/recipe_modules/flavor/android.py @@ -31,6 +31,7 @@ class AndroidFlavor(default.DefaultFlavor): perf_data_dir = android_data_dir + 'perf', resource_dir = android_data_dir + 'resources', images_dir = android_data_dir + 'images', + lotties_dir = android_data_dir + 'lotties', skp_dir = android_data_dir + 'skps', svg_dir = android_data_dir + 'svgs', tmp_dir = android_data_dir) diff --git a/infra/bots/recipe_modules/flavor/api.py b/infra/bots/recipe_modules/flavor/api.py index 9dee6f77ee..090e04c09f 100644 --- a/infra/bots/recipe_modules/flavor/api.py +++ b/infra/bots/recipe_modules/flavor/api.py @@ -28,6 +28,7 @@ commands may be run through ADB. """ +VERSION_FILE_LOTTIE = 'LOTTIE_VERSION' VERSION_FILE_SK_IMAGE = 'SK_IMAGE_VERSION' VERSION_FILE_SKP = 'SKP_VERSION' VERSION_FILE_SVG = 'SVG_VERSION' @@ -106,10 +107,8 @@ class SkiaFlavorApi(recipe_api.RecipeApi): def remove_file_on_device(self, path): return self._f.remove_file_on_device(path) - def install_everything(self): - self.install(skps=True, images=True, svgs=True, resources=True) - - def install(self, skps=False, images=False, svgs=False, resources=False): + def install(self, skps=False, images=False, lotties=False, svgs=False, + resources=False): self._f.install() # TODO(borenet): Only copy files which have changed. @@ -122,6 +121,8 @@ class SkiaFlavorApi(recipe_api.RecipeApi): self._copy_skps() if images: self._copy_images() + if lotties: + self._copy_lotties() if svgs: self._copy_svgs() @@ -150,7 +151,7 @@ class SkiaFlavorApi(recipe_api.RecipeApi): self.copy_file_to_device(actual_version_file, device_version_file) def _copy_images(self): - """Download and copy test images if needed.""" + """Copy test images if needed.""" version = self.m.run.asset_version('skimage', self._skia_dir) self.m.run.writefile( self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_SK_IMAGE), @@ -163,8 +164,22 @@ class SkiaFlavorApi(recipe_api.RecipeApi): self.device_dirs.images_dir) return version + def _copy_lotties(self): + """Copy test lotties if needed.""" + version = self.m.run.asset_version('lottie-samples', self._skia_dir) + self.m.run.writefile( + self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_LOTTIE), + version) + self._copy_dir( + version, + VERSION_FILE_LOTTIE, + self.m.vars.tmp_dir, + self.host_dirs.lotties_dir, + self.device_dirs.lotties_dir) + return version + def _copy_skps(self): - """Download and copy the SKPs if needed.""" + """Copy the SKPs if needed.""" version = self.m.run.asset_version('skp', self._skia_dir) self.m.run.writefile( self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_SKP), @@ -178,7 +193,7 @@ class SkiaFlavorApi(recipe_api.RecipeApi): return version def _copy_svgs(self): - """Download and copy the SVGs if needed.""" + """Copy the SVGs if needed.""" version = self.m.run.asset_version('svg', self._skia_dir) self.m.run.writefile( self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_SVG), diff --git a/infra/bots/recipe_modules/flavor/chromebook.py b/infra/bots/recipe_modules/flavor/chromebook.py index 348f9423a8..2dafd098e3 100644 --- a/infra/bots/recipe_modules/flavor/chromebook.py +++ b/infra/bots/recipe_modules/flavor/chromebook.py @@ -25,6 +25,7 @@ class ChromebookFlavor(default.DefaultFlavor): perf_data_dir = self.chromeos_homedir + 'perf', resource_dir = self.chromeos_homedir + 'resources', images_dir = self.chromeos_homedir + 'images', + lotties_dir = self.chromeos_homedir + 'lotties', skp_dir = self.chromeos_homedir + 'skps', svg_dir = self.chromeos_homedir + 'svgs', tmp_dir = self.chromeos_homedir) diff --git a/infra/bots/recipe_modules/flavor/chromecast.py b/infra/bots/recipe_modules/flavor/chromecast.py index 29c65757cd..9531ff2269 100644 --- a/infra/bots/recipe_modules/flavor/chromecast.py +++ b/infra/bots/recipe_modules/flavor/chromecast.py @@ -28,6 +28,7 @@ class ChromecastFlavor(android.AndroidFlavor): perf_data_dir = data_dir + 'perf', resource_dir = data_dir + 'resources', images_dir = data_dir + 'images', + lotties_dir = data_dir + 'lotties', skp_dir = data_dir + 'skps', svg_dir = data_dir + 'svgs', tmp_dir = data_dir) diff --git a/infra/bots/recipe_modules/flavor/default.py b/infra/bots/recipe_modules/flavor/default.py index b45f489bdf..cfe955e1c3 100644 --- a/infra/bots/recipe_modules/flavor/default.py +++ b/infra/bots/recipe_modules/flavor/default.py @@ -19,6 +19,7 @@ class DeviceDirs(object): perf_data_dir, resource_dir, images_dir, + lotties_dir, skp_dir, svg_dir, tmp_dir): @@ -27,6 +28,7 @@ class DeviceDirs(object): self._perf_data_dir = perf_data_dir self._resource_dir = resource_dir self._images_dir = images_dir + self._lotties_dir = lotties_dir self._skp_dir = skp_dir self._svg_dir = svg_dir self._tmp_dir = tmp_dir @@ -52,6 +54,10 @@ class DeviceDirs(object): def images_dir(self): return self._images_dir + @property + def lotties_dir(self): + return self._lotties_dir + @property def skp_dir(self): """Holds SKP files that are consumed by RenderSKPs and BenchPictures.""" @@ -83,6 +89,7 @@ class DefaultFlavor(object): perf_data_dir=self.m.vars.swarming_out_dir, resource_dir=self.m.path['start_dir'].join('skia', 'resources'), images_dir=self.m.path['start_dir'].join('skimage'), + lotties_dir=self.m.path['start_dir'].join('lottie-samples'), skp_dir=self.m.path['start_dir'].join('skp'), svg_dir=self.m.path['start_dir'].join('svg'), tmp_dir=self.m.vars.tmp_dir) diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-Lottie.json b/infra/bots/recipe_modules/flavor/examples/full.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-Lottie.json new file mode 100644 index 0000000000..84de0c1b44 --- /dev/null +++ b/infra/bots/recipe_modules/flavor/examples/full.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-Lottie.json @@ -0,0 +1,108 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "results_dir" + ], + "infra_step": true, + "name": "rmtree results_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "results_dir" + ], + "infra_step": true, + "name": "makedirs results_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "device_results_dir" + ], + "infra_step": true, + "name": "rmtree device_results_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "device_results_dir" + ], + "infra_step": true, + "name": "makedirs device_results_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/skia/infra/bots/assets/lottie-samples/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get lottie-samples VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "42", + "[START_DIR]/tmp/LOTTIE_VERSION" + ], + "infra_step": true, + "name": "write LOTTIE_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::flavor]/resources/symbolize_stack_trace.py", + "[START_DIR]", + "catchsegv", + "[START_DIR]/build/dm", + "--some-flag" + ], + "cwd": "[START_DIR]/skia", + "env": { + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "symbolized dm" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipe_modules/flavor/examples/full.py b/infra/bots/recipe_modules/flavor/examples/full.py index 3c7cd3dd51..f59c92a520 100644 --- a/infra/bots/recipe_modules/flavor/examples/full.py +++ b/infra/bots/recipe_modules/flavor/examples/full.py @@ -39,7 +39,11 @@ def RunSteps(api): api.flavor.copy_file_to_device('file.txt', 'file.txt') api.flavor.create_clean_host_dir('results_dir') api.flavor.create_clean_device_dir('device_results_dir') - api.flavor.install_everything() + if 'Lottie' in api.properties['buildername']: + api.flavor.install(lotties=True) + else: + api.flavor.install(skps=True, images=True, lotties=False, svgs=True, + resources=True) if 'Test' in api.properties['buildername']: api.flavor.step('dm', ['dm', '--some-flag']) api.flavor.copy_directory_contents_to_host( @@ -67,6 +71,7 @@ TEST_BUILDERS = [ 'Test-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Release-All-Android_ASAN', 'Test-ChromeOS-Clang-SamsungChromebookPlus-GPU-MaliT860-arm-Release-All', 'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-Coverage', + 'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-Lottie', 'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-TSAN', 'Test-Debian9-Clang-GCE-GPU-SwiftShader-x86_64-Debug-All-SwiftShader', 'Test-Debian9-Clang-NUC7i5BNK-GPU-IntelIris640-x86_64-Debug-All-OpenCL', diff --git a/infra/bots/recipe_modules/flavor/ios.py b/infra/bots/recipe_modules/flavor/ios.py index f62de045c3..9e21a7dc7f 100644 --- a/infra/bots/recipe_modules/flavor/ios.py +++ b/infra/bots/recipe_modules/flavor/ios.py @@ -21,6 +21,7 @@ class iOSFlavor(default.DefaultFlavor): perf_data_dir='perf', resource_dir='resources', images_dir='images', + lotties_dir='lotties', skp_dir='skps', svg_dir='svgs', tmp_dir='tmp') diff --git a/infra/bots/recipes/perf.py b/infra/bots/recipes/perf.py index ed3e82dcbf..9eadca5c5f 100644 --- a/infra/bots/recipes/perf.py +++ b/infra/bots/recipes/perf.py @@ -352,7 +352,7 @@ def RunSteps(api): if 'Chromecast' in api.vars.builder_name: api.flavor.install(resources=True, skps=True) else: - api.flavor.install_everything() + api.flavor.install(skps=True, images=True, svgs=True, resources=True) perf_steps(api) finally: api.flavor.cleanup_steps() diff --git a/infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-Lottie.json b/infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-Lottie.json new file mode 100644 index 0000000000..c55196ecba --- /dev/null +++ b/infra/bots/recipes/test.expected/Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-Lottie.json @@ -0,0 +1,478 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/tmp" + ], + "infra_step": true, + "name": "makedirs tmp_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/skia/infra/bots/assets/lottie-samples/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get lottie-samples VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "42", + "[START_DIR]/tmp/LOTTIE_VERSION" + ], + "infra_step": true, + "name": "write LOTTIE_VERSION" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "rmtree", + "[START_DIR]/test" + ], + "infra_step": true, + "name": "rmtree test" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/test" + ], + "infra_step": true, + "name": "makedirs test" + }, + { + "cmd": [ + "python", + "-u", + "\nimport contextlib\nimport math\nimport socket\nimport sys\nimport time\nimport urllib2\n\nHASHES_URL = 'https://storage.googleapis.com/skia-infra-gm/hash_files/gold-prod-hashes.txt'\nRETRIES = 5\nTIMEOUT = 60\nWAIT_BASE = 15\n\nsocket.setdefaulttimeout(TIMEOUT)\nfor retry in range(RETRIES):\n try:\n with contextlib.closing(\n urllib2.urlopen(HASHES_URL, timeout=TIMEOUT)) as w:\n hashes = w.read()\n with open(sys.argv[1], 'w') as f:\n f.write(hashes)\n break\n except Exception as e:\n print 'Failed to get uninteresting hashes from %s:' % HASHES_URL\n print e\n if retry == RETRIES:\n raise\n waittime = WAIT_BASE * math.pow(2, retry)\n print 'Retry in %d seconds.' % waittime\n time.sleep(waittime)\n", + "[START_DIR]/tmp/uninteresting_hashes.txt" + ], + "env": { + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]" + }, + "infra_step": true, + "name": "get uninteresting hashes", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import contextlib@@@", + "@@@STEP_LOG_LINE@python.inline@import math@@@", + "@@@STEP_LOG_LINE@python.inline@import socket@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@import time@@@", + "@@@STEP_LOG_LINE@python.inline@import urllib2@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@HASHES_URL = 'https://storage.googleapis.com/skia-infra-gm/hash_files/gold-prod-hashes.txt'@@@", + "@@@STEP_LOG_LINE@python.inline@RETRIES = 5@@@", + "@@@STEP_LOG_LINE@python.inline@TIMEOUT = 60@@@", + "@@@STEP_LOG_LINE@python.inline@WAIT_BASE = 15@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@socket.setdefaulttimeout(TIMEOUT)@@@", + "@@@STEP_LOG_LINE@python.inline@for retry in range(RETRIES):@@@", + "@@@STEP_LOG_LINE@python.inline@ try:@@@", + "@@@STEP_LOG_LINE@python.inline@ with contextlib.closing(@@@", + "@@@STEP_LOG_LINE@python.inline@ urllib2.urlopen(HASHES_URL, timeout=TIMEOUT)) as w:@@@", + "@@@STEP_LOG_LINE@python.inline@ hashes = w.read()@@@", + "@@@STEP_LOG_LINE@python.inline@ with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ f.write(hashes)@@@", + "@@@STEP_LOG_LINE@python.inline@ break@@@", + "@@@STEP_LOG_LINE@python.inline@ except Exception as e:@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Failed to get uninteresting hashes from %s:' % HASHES_URL@@@", + "@@@STEP_LOG_LINE@python.inline@ print e@@@", + "@@@STEP_LOG_LINE@python.inline@ if retry == RETRIES:@@@", + "@@@STEP_LOG_LINE@python.inline@ raise@@@", + "@@@STEP_LOG_LINE@python.inline@ waittime = WAIT_BASE * math.pow(2, retry)@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Retry in %d seconds.' % waittime@@@", + "@@@STEP_LOG_LINE@python.inline@ time.sleep(waittime)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n" + ], + "name": "get swarming bot id", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n" + ], + "name": "get swarming task id", + "stdout": "/path/to/tmp/", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[skia::flavor]/resources/symbolize_stack_trace.py", + "[START_DIR]", + "catchsegv", + "[START_DIR]/build/dm", + "--resourcePath", + "[START_DIR]/skia/resources", + "--skps", + "[START_DIR]/skp", + "--images", + "[START_DIR]/skimage/dm", + "--colorImages", + "[START_DIR]/skimage/colorspace", + "--nameByHash", + "--properties", + "gitHash", + "abc123", + "builder", + "Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-Lottie", + "buildbucket_build_id", + "123454321", + "swarming_bot_id", + "skia-bot-123", + "swarming_task_id", + "123456", + "--svgs", + "[START_DIR]/svg", + "--lotties", + "[START_DIR]/lottie-samples", + "--key", + "arch", + "x86_64", + "compiler", + "Clang", + "configuration", + "Release", + "cpu_or_gpu", + "CPU", + "cpu_or_gpu_value", + "AVX2", + "extra_config", + "Lottie", + "model", + "GCE", + "os", + "Debian9", + "--uninterestingHashesFile", + "[START_DIR]/tmp/uninteresting_hashes.txt", + "--writePath", + "[START_DIR]/[SWARM_OUT_DIR]", + "--dont_write", + "pdf", + "--randomProcessorTest", + "--nogpu", + "--config", + "8888", + "pdf", + "g8", + "565", + "lite-8888", + "gbr-8888", + "f16", + "srgb", + "esrgb", + "narrow", + "enarrow", + "serialize-8888", + "tiles_rt-8888", + "pic-8888", + "--src", + "lottie", + "--blacklist", + "f16", + "_", + "_", + "dstreadshuffle", + "gbr-8888", + "image", + "_", + "_", + "gbr-8888", + "colorImage", + "_", + "_", + "g8", + "image", + "_", + "_", + "g8", + "colorImage", + "_", + "_", + "serialize-8888", + "gm", + "_", + "bleed_image", + "serialize-8888", + "gm", + "_", + "c_gms", + "serialize-8888", + "gm", + "_", + "colortype", + "serialize-8888", + "gm", + "_", + "colortype_xfermodes", + "serialize-8888", + "gm", + "_", + "drawfilter", + "serialize-8888", + "gm", + "_", + "fontmgr_bounds_0.75_0", + "serialize-8888", + "gm", + "_", + "fontmgr_bounds_1_-0.25", + "serialize-8888", + "gm", + "_", + "fontmgr_bounds", + "serialize-8888", + "gm", + "_", + "fontmgr_match", + "serialize-8888", + "gm", + "_", + "fontmgr_iter", + "serialize-8888", + "gm", + "_", + "imagemasksubset", + "serialize-8888", + "gm", + "_", + "bitmapfilters", + "serialize-8888", + "gm", + "_", + "bitmapshaders", + "serialize-8888", + "gm", + "_", + "bleed", + "serialize-8888", + "gm", + "_", + "bleed_alpha_bmp", + "serialize-8888", + "gm", + "_", + "bleed_alpha_bmp_shader", + "serialize-8888", + "gm", + "_", + "convex_poly_clip", + "serialize-8888", + "gm", + "_", + "extractalpha", + "serialize-8888", + "gm", + "_", + "filterbitmap_checkerboard_32_32_g8", + "serialize-8888", + "gm", + "_", + "filterbitmap_image_mandrill_64", + "serialize-8888", + "gm", + "_", + "shadows", + "serialize-8888", + "gm", + "_", + "simpleaaclip_aaclip", + "serialize-8888", + "gm", + "_", + "composeshader_bitmap", + "serialize-8888", + "gm", + "_", + "scaled_tilemodes_npot", + "serialize-8888", + "gm", + "_", + "scaled_tilemodes", + "serialize-8888", + "gm", + "_", + "typefacerendering_pfaMac", + "serialize-8888", + "gm", + "_", + "parsedpaths", + "serialize-8888", + "gm", + "_", + "ImageGeneratorExternal_rect", + "serialize-8888", + "gm", + "_", + "ImageGeneratorExternal_shader", + "serialize-8888", + "gm", + "_", + "shadow_utils", + "serialize-8888", + "gm", + "_", + "persp_images", + "serialize-8888", + "gm", + "_", + "all_bitmap_configs", + "serialize-8888", + "gm", + "_", + "makecolorspace", + "serialize-8888", + "gm", + "_", + "analytic_antialias_convex", + "serialize-8888", + "gm", + "_", + "bleed_alpha_image", + "serialize-8888", + "gm", + "_", + "bleed_alpha_image_shader", + "pic-8888", + "gm", + "_", + "drawfilter", + "lite-8888", + "gm", + "_", + "drawfilter", + "pic-8888", + "gm", + "_", + "image-cacherator-from-picture", + "serialize-8888", + "gm", + "_", + "image-cacherator-from-picture", + "pic-8888", + "gm", + "_", + "image-cacherator-from-raster", + "serialize-8888", + "gm", + "_", + "image-cacherator-from-raster", + "pic-8888", + "gm", + "_", + "image-cacherator-from-ctable", + "serialize-8888", + "gm", + "_", + "image-cacherator-from-ctable", + "pic-8888", + "gm", + "_", + "gamut", + "lite-8888", + "gm", + "_", + "gamut", + "serialize-8888", + "gm", + "_", + "gamut", + "pic-8888", + "gm", + "_", + "complexclip4_bw", + "lite-8888", + "gm", + "_", + "complexclip4_bw", + "serialize-8888", + "gm", + "_", + "complexclip4_bw", + "pic-8888", + "gm", + "_", + "complexclip4_aa", + "lite-8888", + "gm", + "_", + "complexclip4_aa", + "serialize-8888", + "gm", + "_", + "complexclip4_aa", + "tiles_rt-8888", + "gm", + "_", + "complexclip4_bw", + "tiles_rt-8888", + "gm", + "_", + "complexclip4_aa", + "--nonativeFonts", + "--verbose" + ], + "cwd": "[START_DIR]/skia", + "env": { + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "symbolized dm" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipes/test.py b/infra/bots/recipes/test.py index c95e92a9d2..8ef50581a2 100644 --- a/infra/bots/recipes/test.py +++ b/infra/bots/recipes/test.py @@ -11,7 +11,6 @@ DEPS = [ 'flavor', 'recipe_engine/context', 'recipe_engine/file', - 'recipe_engine/json', 'recipe_engine/path', 'recipe_engine/platform', 'recipe_engine/properties', @@ -301,7 +300,7 @@ def dm_flags(api, bot): args.extend(configs) # Run tests, gms, and image decoding tests everywhere. - args.extend('--src tests gm image colorImage svg skp'.split(' ')) + args.extend('--src tests gm image lottie colorImage svg skp'.split(' ')) if api.vars.builder_cfg.get('cpu_or_gpu') == 'GPU': # Don't run the 'svgparse_*' svgs on GPU. blacklist('_ svg _ svgparse_') @@ -325,15 +324,30 @@ def dm_flags(api, bot): args.remove('image') args.remove('colorImage') + def remove_from_args(arg): + if arg in args: + args.remove(arg) + if 'DDL' in bot: # The DDL bots just render the large skps and the gms - args.remove('tests') - args.remove('image') - args.remove('colorImage') - args.remove('svg') + remove_from_args('tests') + remove_from_args('image') + remove_from_args('colorImage') + remove_from_args('svg') else: # Currently, only the DDL bots render skps - args.remove('skp') + remove_from_args('skp') + + if 'Lottie' in api.vars.builder_cfg.get('extra_config', ''): + # Only run the lotties on Lottie bots. + remove_from_args('tests') + remove_from_args('gm') + remove_from_args('image') + remove_from_args('colorImage') + remove_from_args('svg') + remove_from_args('skp') + else: + remove_from_args('lottie') # TODO: ??? blacklist('f16 _ _ dstreadshuffle') @@ -948,6 +962,8 @@ def test_steps(api): ] + properties args.extend(['--svgs', api.flavor.device_dirs.svg_dir]) + if 'Lottie' in api.vars.builder_cfg.get('extra_config', ''): + args.extend(['--lotties', api.flavor.device_dirs.lotties_dir]) args.append('--key') args.extend(key_params(api)) @@ -988,8 +1004,10 @@ def RunSteps(api): try: if 'Chromecast' in api.vars.builder_name: api.flavor.install(resources=True, skps=True) + elif 'Lottie' in api.vars.builder_name: + api.flavor.install(resources=True, lotties=True) else: - api.flavor.install_everything() + api.flavor.install(skps=True, images=True, svgs=True, resources=True) test_steps(api) finally: api.flavor.cleanup_steps() @@ -1020,6 +1038,7 @@ TEST_BUILDERS = [ ('Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All' '-SK_USE_DISCARDABLE_SCALEDIMAGECACHE'), 'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-T8888', + 'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-Lottie', ('Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All' '-SK_FORCE_RASTER_PIPELINE_BLITTER'), 'Test-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-TSAN', -- cgit v1.2.3