diff options
author | Eric Boren <borenet@google.com> | 2018-05-24 09:14:18 -0400 |
---|---|---|
committer | Skia Commit-Bot <skia-commit-bot@chromium.org> | 2018-05-24 14:06:55 +0000 |
commit | 90f050387a5851943910f91dce2c01babf0d3896 (patch) | |
tree | 9846116e12f46b08d0c2ac2b3ad556a4898814ae /infra | |
parent | 2841362222122115abc06d3ede97523e3044ea7b (diff) |
[recipes] Rename some modules and files
- Move doxygen and binary size out of core and into their own modules.
- Rename core -> checkout since that's all it does, shorten method names.
- Rename flavors:
- Everything is GN, so remove GN/gn_ everywhere.
- Merge gn_flavor into default.
- Shorten file / module names.
Bug: skia:6473
Change-Id: I8ac9ff9c9a267f366206b9991adfa5eb37126ca7
Reviewed-on: https://skia-review.googlesource.com/129176
Commit-Queue: Eric Boren <borenet@google.com>
Reviewed-by: Ben Wagner <benjaminwagner@google.com>
Diffstat (limited to 'infra')
48 files changed, 412 insertions, 286 deletions
diff --git a/infra/bots/recipe_modules/binary_size/__init__.py b/infra/bots/recipe_modules/binary_size/__init__.py new file mode 100644 index 0000000000..4e3b671627 --- /dev/null +++ b/infra/bots/recipe_modules/binary_size/__init__.py @@ -0,0 +1,11 @@ +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +DEPS = [ + 'recipe_engine/context', + 'recipe_engine/properties', + 'recipe_engine/step', + 'run', + 'vars', +] diff --git a/infra/bots/recipe_modules/binary_size/api.py b/infra/bots/recipe_modules/binary_size/api.py new file mode 100644 index 0000000000..db8ee9d3a9 --- /dev/null +++ b/infra/bots/recipe_modules/binary_size/api.py @@ -0,0 +1,22 @@ +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +from recipe_engine import recipe_api +from recipe_engine import config_types + + +class BinarySizeApi(recipe_api.RecipeApi): + def run_analysis(self, skia_dir, dest_file): + cmd = ['python', self.resource('run_binary_size_analysis.py'), + '--library', self.m.vars.skia_out.join('libskia.so'), + '--githash', self.m.properties['revision'], + '--dest', dest_file] + if self.m.vars.is_trybot: + cmd.extend(['--issue_number', str(self.m.properties['patch_issue'])]) + with self.m.context(cwd=skia_dir): + self.m.run( + self.m.step, + 'generate binary size data', + cmd=cmd) diff --git a/infra/bots/recipe_modules/binary_size/examples/full.expected/binary_size.json b/infra/bots/recipe_modules/binary_size/examples/full.expected/binary_size.json new file mode 100644 index 0000000000..a5dd9413a5 --- /dev/null +++ b/infra/bots/recipe_modules/binary_size/examples/full.expected/binary_size.json @@ -0,0 +1,26 @@ +[ + { + "cmd": [ + "python", + "RECIPE_MODULE[skia::binary_size]/resources/run_binary_size_analysis.py", + "--library", + "[START_DIR]/build/out/Release/libskia.so", + "--githash", + "abc123", + "--dest", + "[START_DIR]/binary_size", + "--issue_number", + "456789" + ], + "env": { + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "generate binary size data" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/binary_size/examples/full.py b/infra/bots/recipe_modules/binary_size/examples/full.py new file mode 100644 index 0000000000..135fedc8e4 --- /dev/null +++ b/infra/bots/recipe_modules/binary_size/examples/full.py @@ -0,0 +1,33 @@ +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +DEPS = [ + 'binary_size', + 'recipe_engine/path', + 'recipe_engine/properties', + 'vars', +] + + +def RunSteps(api): + api.vars.setup() + dest_file = api.path['start_dir'].join('binary_size') + api.binary_size.run_analysis(api.path['start_dir'], dest_file) + + +def GenTests(api): + yield ( + api.test('binary_size') + + api.properties(buildername='Housekeeper-PerCommit', + repository='https://skia.googlesource.com/skia.git', + revision='abc123', + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + + api.properties.tryserver( + buildername='Housekeeper-PerCommit', + gerrit_project='skia', + gerrit_url='https://skia-review.googlesource.com/', + ) + ) diff --git a/infra/bots/recipe_modules/core/resources/binary_size_utils.py b/infra/bots/recipe_modules/binary_size/resources/binary_size_utils.py index c09a65dccd..c09a65dccd 100644 --- a/infra/bots/recipe_modules/core/resources/binary_size_utils.py +++ b/infra/bots/recipe_modules/binary_size/resources/binary_size_utils.py diff --git a/infra/bots/recipe_modules/core/resources/elf_symbolizer.py b/infra/bots/recipe_modules/binary_size/resources/elf_symbolizer.py index de9c141219..de9c141219 100644 --- a/infra/bots/recipe_modules/core/resources/elf_symbolizer.py +++ b/infra/bots/recipe_modules/binary_size/resources/elf_symbolizer.py diff --git a/infra/bots/recipe_modules/core/resources/run_binary_size_analysis.py b/infra/bots/recipe_modules/binary_size/resources/run_binary_size_analysis.py index 822b366614..822b366614 100755 --- a/infra/bots/recipe_modules/core/resources/run_binary_size_analysis.py +++ b/infra/bots/recipe_modules/binary_size/resources/run_binary_size_analysis.py diff --git a/infra/bots/recipe_modules/core/__init__.py b/infra/bots/recipe_modules/checkout/__init__.py index 29f438e3a6..29f438e3a6 100644 --- a/infra/bots/recipe_modules/core/__init__.py +++ b/infra/bots/recipe_modules/checkout/__init__.py diff --git a/infra/bots/recipe_modules/core/api.py b/infra/bots/recipe_modules/checkout/api.py index f943b865b2..67c27b1aeb 100644 --- a/infra/bots/recipe_modules/core/api.py +++ b/infra/bots/recipe_modules/checkout/api.py @@ -6,16 +6,11 @@ # pylint: disable=W0201 -import json -import os -import re -import sys - from recipe_engine import recipe_api from recipe_engine import config_types -class SkiaApi(recipe_api.RecipeApi): +class CheckoutApi(recipe_api.RecipeApi): @property def default_checkout_root(self): @@ -26,7 +21,7 @@ class SkiaApi(recipe_api.RecipeApi): """Build a ref for the given issue and patchset.""" return 'refs/changes/%s/%s/%s' % (issue[-2:], issue, patchset) - def checkout_git(self, checkout_root): + def git(self, checkout_root): """Run the steps to perform a pure-git checkout without DEPS.""" skia_dir = checkout_root.join('skia') self.m.git.checkout( @@ -39,7 +34,7 @@ class SkiaApi(recipe_api.RecipeApi): self.m.git('rebase', self.m.properties['revision']) return self.m.properties['revision'] - def checkout_bot_update(self, checkout_root, gclient_cache=None): + def bot_update(self, checkout_root, gclient_cache=None): """Run the steps to obtain a checkout using bot_update.""" if not gclient_cache: gclient_cache = self.m.vars.cache_dir.join('git') diff --git a/infra/bots/recipe_modules/core/examples/full.expected/Build-Debian9-Clang-x86_64-Release-NoDEPS.json b/infra/bots/recipe_modules/checkout/examples/full.expected/Build-Debian9-Clang-x86_64-Release-NoDEPS.json index ca21fe033f..ca21fe033f 100644 --- a/infra/bots/recipe_modules/core/examples/full.expected/Build-Debian9-Clang-x86_64-Release-NoDEPS.json +++ b/infra/bots/recipe_modules/checkout/examples/full.expected/Build-Debian9-Clang-x86_64-Release-NoDEPS.json diff --git a/infra/bots/recipe_modules/core/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json b/infra/bots/recipe_modules/checkout/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json index d9db2bbf14..d9db2bbf14 100644 --- a/infra/bots/recipe_modules/core/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json +++ b/infra/bots/recipe_modules/checkout/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json diff --git a/infra/bots/recipe_modules/core/examples/full.expected/Build-Win-Clang-x86_64-Release-ParentRevision.json b/infra/bots/recipe_modules/checkout/examples/full.expected/Build-Win-Clang-x86_64-Release-ParentRevision.json index ffce3b2ee9..ffce3b2ee9 100644 --- a/infra/bots/recipe_modules/core/examples/full.expected/Build-Win-Clang-x86_64-Release-ParentRevision.json +++ b/infra/bots/recipe_modules/checkout/examples/full.expected/Build-Win-Clang-x86_64-Release-ParentRevision.json diff --git a/infra/bots/recipe_modules/core/examples/full.expected/Housekeeper-Weekly-RecreateSKPs.json b/infra/bots/recipe_modules/checkout/examples/full.expected/Housekeeper-Weekly-RecreateSKPs.json index f3980c9d00..f3980c9d00 100644 --- a/infra/bots/recipe_modules/core/examples/full.expected/Housekeeper-Weekly-RecreateSKPs.json +++ b/infra/bots/recipe_modules/checkout/examples/full.expected/Housekeeper-Weekly-RecreateSKPs.json diff --git a/infra/bots/recipe_modules/core/examples/full.expected/cross_repo_trybot.json b/infra/bots/recipe_modules/checkout/examples/full.expected/cross_repo_trybot.json index 30f707cec4..30f707cec4 100644 --- a/infra/bots/recipe_modules/core/examples/full.expected/cross_repo_trybot.json +++ b/infra/bots/recipe_modules/checkout/examples/full.expected/cross_repo_trybot.json diff --git a/infra/bots/recipe_modules/core/examples/full.expected/flutter_trybot.json b/infra/bots/recipe_modules/checkout/examples/full.expected/flutter_trybot.json index 85cbb107ab..85cbb107ab 100644 --- a/infra/bots/recipe_modules/core/examples/full.expected/flutter_trybot.json +++ b/infra/bots/recipe_modules/checkout/examples/full.expected/flutter_trybot.json diff --git a/infra/bots/recipe_modules/core/examples/full.expected/parent_revision_trybot.json b/infra/bots/recipe_modules/checkout/examples/full.expected/parent_revision_trybot.json index 2a1eb4cb64..2a1eb4cb64 100644 --- a/infra/bots/recipe_modules/core/examples/full.expected/parent_revision_trybot.json +++ b/infra/bots/recipe_modules/checkout/examples/full.expected/parent_revision_trybot.json diff --git a/infra/bots/recipe_modules/core/examples/full.expected/test.json b/infra/bots/recipe_modules/checkout/examples/full.expected/test.json index ef0d4965bc..ef0d4965bc 100644 --- a/infra/bots/recipe_modules/core/examples/full.expected/test.json +++ b/infra/bots/recipe_modules/checkout/examples/full.expected/test.json diff --git a/infra/bots/recipe_modules/core/examples/full.py b/infra/bots/recipe_modules/checkout/examples/full.py index 2d1279f547..7e7f436783 100644 --- a/infra/bots/recipe_modules/core/examples/full.py +++ b/infra/bots/recipe_modules/checkout/examples/full.py @@ -4,7 +4,7 @@ DEPS = [ - 'core', + 'checkout', 'recipe_engine/file', 'recipe_engine/path', 'recipe_engine/properties', @@ -21,12 +21,12 @@ def RunSteps(api): bot_update = False if bot_update: - checkout_root = api.core.default_checkout_root + checkout_root = api.checkout.default_checkout_root if 'Flutter' in api.vars.builder_name: checkout_root = checkout_root.join('flutter') - api.core.checkout_bot_update(checkout_root=checkout_root) + api.checkout.bot_update(checkout_root=checkout_root) else: - api.core.checkout_git(checkout_root=api.path['start_dir']) + api.checkout.git(checkout_root=api.path['start_dir']) api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) diff --git a/infra/bots/recipe_modules/doxygen/__init__.py b/infra/bots/recipe_modules/doxygen/__init__.py new file mode 100644 index 0000000000..1825f6116a --- /dev/null +++ b/infra/bots/recipe_modules/doxygen/__init__.py @@ -0,0 +1,9 @@ +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +DEPS = [ + 'recipe_engine/context', + 'recipe_engine/step', + 'run', +] diff --git a/infra/bots/recipe_modules/doxygen/api.py b/infra/bots/recipe_modules/doxygen/api.py new file mode 100644 index 0000000000..6141556fe0 --- /dev/null +++ b/infra/bots/recipe_modules/doxygen/api.py @@ -0,0 +1,17 @@ +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +from recipe_engine import recipe_api +from recipe_engine import config_types + + +class DoxygenApi(recipe_api.RecipeApi): + def generate_and_upload(self, skia_dir): + with self.m.context(cwd=skia_dir): + self.m.run( + self.m.step, + 'generate and upload doxygen', + cmd=['python', self.resource('generate_and_upload_doxygen.py')], + abort_on_failure=False) diff --git a/infra/bots/recipe_modules/doxygen/examples/full.expected/doxygen.json b/infra/bots/recipe_modules/doxygen/examples/full.expected/doxygen.json new file mode 100644 index 0000000000..bef199f5a1 --- /dev/null +++ b/infra/bots/recipe_modules/doxygen/examples/full.expected/doxygen.json @@ -0,0 +1,18 @@ +[ + { + "cmd": [ + "python", + "RECIPE_MODULE[skia::doxygen]/resources/generate_and_upload_doxygen.py" + ], + "env": { + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "generate and upload doxygen" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/doxygen/examples/full.py b/infra/bots/recipe_modules/doxygen/examples/full.py new file mode 100644 index 0000000000..c7335fa41c --- /dev/null +++ b/infra/bots/recipe_modules/doxygen/examples/full.py @@ -0,0 +1,27 @@ +# Copyright 2018 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +DEPS = [ + 'doxygen', + 'recipe_engine/path', + 'recipe_engine/properties', + 'vars', +] + + +def RunSteps(api): + api.vars.setup() + api.doxygen.generate_and_upload(api.path['start_dir']) + + +def GenTests(api): + yield ( + api.test('doxygen') + + api.properties(buildername='Housekeeper-PerCommit', + repository='https://skia.googlesource.com/skia.git', + revision='abc123', + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + ) diff --git a/infra/bots/recipe_modules/core/resources/generate_and_upload_doxygen.py b/infra/bots/recipe_modules/doxygen/resources/generate_and_upload_doxygen.py index 968f80debf..968f80debf 100755 --- a/infra/bots/recipe_modules/core/resources/generate_and_upload_doxygen.py +++ b/infra/bots/recipe_modules/doxygen/resources/generate_and_upload_doxygen.py diff --git a/infra/bots/recipe_modules/flavor/__init__.py b/infra/bots/recipe_modules/flavor/__init__.py index 37321e559a..28b3d9bacb 100644 --- a/infra/bots/recipe_modules/flavor/__init__.py +++ b/infra/bots/recipe_modules/flavor/__init__.py @@ -12,6 +12,7 @@ DEPS = [ 'infra', 'recipe_engine/context', 'recipe_engine/file', + 'recipe_engine/json', 'recipe_engine/path', 'recipe_engine/platform', 'recipe_engine/python', diff --git a/infra/bots/recipe_modules/flavor/gn_android_flavor.py b/infra/bots/recipe_modules/flavor/android.py index 1c352afc28..ed8d9af2d4 100644 --- a/infra/bots/recipe_modules/flavor/gn_android_flavor.py +++ b/infra/bots/recipe_modules/flavor/android.py @@ -2,17 +2,19 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + from recipe_engine import recipe_api -import default_flavor -import re -import subprocess +from . import default +import subprocess # TODO(borenet): No! Remove this. + + +"""Android flavor, used for running code on Android.""" -"""GN Android flavor utils, used for building Skia for Android with GN.""" -class GNAndroidFlavorUtils(default_flavor.DefaultFlavorUtils): +class AndroidFlavor(default.DefaultFlavor): def __init__(self, m): - super(GNAndroidFlavorUtils, self).__init__(m) + super(AndroidFlavor, self).__init__(m) self._ever_ran_adb = False self.ADB_BINARY = '/usr/bin/adb.1.0.35' self.ADB_PUB_KEY = '/home/chrome-bot/.android/adbkey' @@ -24,7 +26,7 @@ class GNAndroidFlavorUtils(default_flavor.DefaultFlavorUtils): # Data should go in android_data_dir, which may be preserved across runs. android_data_dir = '/sdcard/revenge_of_the_skiabot/' - self.device_dirs = default_flavor.DeviceDirs( + self.device_dirs = default.DeviceDirs( bin_dir = '/data/local/tmp/', dm_dir = android_data_dir + 'dm_out', perf_data_dir = android_data_dir + 'perf', diff --git a/infra/bots/recipe_modules/flavor/api.py b/infra/bots/recipe_modules/flavor/api.py index c5a1be8030..9dee6f77ee 100644 --- a/infra/bots/recipe_modules/flavor/api.py +++ b/infra/bots/recipe_modules/flavor/api.py @@ -8,13 +8,24 @@ from recipe_engine import recipe_api -from . import default_flavor -from . import gn_android_flavor -from . import gn_chromebook_flavor -from . import gn_chromecast_flavor -from . import gn_flavor -from . import ios_flavor -from . import valgrind_flavor +from . import android +from . import chromebook +from . import chromecast +from . import default +from . import ios +from . import valgrind + + +"""Abstractions for running code on various platforms. + +The methods in this module define how certain high-level functions should work. +Each flavor should correspond to a subclass of DefaultFlavor which may override +any of these functions as appropriate for that flavor. + +For example, the AndroidFlavor will override the functions for copying files +between the host and Android device, as well as the 'step' function, so that +commands may be run through ADB. +""" VERSION_FILE_SK_IMAGE = 'SK_IMAGE_VERSION' @@ -50,17 +61,17 @@ class SkiaFlavorApi(recipe_api.RecipeApi): def get_flavor(self, vars_api): """Return a flavor utils object specific to the given builder.""" if is_chromecast(vars_api): - return gn_chromecast_flavor.GNChromecastFlavorUtils(self) + return chromecast.ChromecastFlavor(self) if is_chromebook(vars_api): - return gn_chromebook_flavor.GNChromebookFlavorUtils(self) + return chromebook.ChromebookFlavor(self) if is_android(vars_api) and not is_test_skqp(vars_api): - return gn_android_flavor.GNAndroidFlavorUtils(self) + return android.AndroidFlavor(self) elif is_ios(vars_api): - return ios_flavor.iOSFlavorUtils(self) + return ios.iOSFlavor(self) elif is_valgrind(vars_api): - return valgrind_flavor.ValgrindFlavorUtils(self) + return valgrind.ValgrindFlavor(self) else: - return gn_flavor.GNFlavorUtils(self) + return default.DefaultFlavor(self) def setup(self): self._f = self.get_flavor(self.m.vars) diff --git a/infra/bots/recipe_modules/flavor/gn_chromebook_flavor.py b/infra/bots/recipe_modules/flavor/chromebook.py index a133f2bb90..eb745d3972 100644 --- a/infra/bots/recipe_modules/flavor/gn_chromebook_flavor.py +++ b/infra/bots/recipe_modules/flavor/chromebook.py @@ -5,24 +5,21 @@ from recipe_engine import recipe_api -import default_flavor -import gn_flavor -import json -import subprocess +import default +import json # TODO(borenet): No! Remove this. -""" - GN Chromebook flavor utils, used for building and testing Skia for ARM - Chromebooks with GN -""" -class GNChromebookFlavorUtils(gn_flavor.GNFlavorUtils): +"""Chromebook flavor, used for running code on Chromebooks.""" + + +class ChromebookFlavor(default.DefaultFlavor): def __init__(self, m): - super(GNChromebookFlavorUtils, self).__init__(m) + super(ChromebookFlavor, self).__init__(m) self._user_ip = '' self.chromeos_homedir = '/home/chronos/user/' - self.device_dirs = default_flavor.DeviceDirs( + self.device_dirs = default.DeviceDirs( bin_dir = self.chromeos_homedir + 'bin', dm_dir = self.chromeos_homedir + 'dm_out', perf_data_dir = self.chromeos_homedir + 'perf', diff --git a/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py b/infra/bots/recipe_modules/flavor/chromecast.py index bd65e01516..c897aa93ab 100644 --- a/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py +++ b/infra/bots/recipe_modules/flavor/chromecast.py @@ -4,15 +4,16 @@ from recipe_engine import recipe_api -import default_flavor -import gn_android_flavor -import subprocess +from . import android +from . import default -"""GN Chromecast flavor utils, used for building Skia for Chromecast with GN""" -class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils): +"""Chromecast flavor, used for running code on Chromecast""" + + +class ChromecastFlavor(android.AndroidFlavor): def __init__(self, m): - super(GNChromecastFlavorUtils, self).__init__(m) + super(ChromecastFlavor, self).__init__(m) self._ever_ran_adb = False self._user_ip = '' @@ -21,7 +22,7 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils): # resources, executable and output the dm images. So, we have dm_out be # on the tempfs (i.e. RAM) /dev/shm. (which is about 140M) data_dir = '/cache/skia/' - self.device_dirs = default_flavor.DeviceDirs( + self.device_dirs = default.DeviceDirs( bin_dir = '/cache/skia/bin', dm_dir = '/dev/shm/skia/dm_out', perf_data_dir = data_dir + 'perf', @@ -51,7 +52,7 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils): return self.user_ip_host.split(':')[0] def install(self): - super(GNChromecastFlavorUtils, self).install() + super(ChromecastFlavor, self).install() self._adb('mkdir ' + self.device_dirs.bin_dir, 'shell', 'mkdir', '-p', self.device_dirs.bin_dir) diff --git a/infra/bots/recipe_modules/flavor/gn_flavor.py b/infra/bots/recipe_modules/flavor/default.py index 284f20b365..501c6bead5 100644 --- a/infra/bots/recipe_modules/flavor/gn_flavor.py +++ b/infra/bots/recipe_modules/flavor/default.py @@ -1,12 +1,147 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. +# Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import default_flavor -"""GN flavor utils, used for building Skia with GN.""" -class GNFlavorUtils(default_flavor.DefaultFlavorUtils): - # TODO(borenet): Delete this file. +# pylint: disable=W0201 + + +"""Default flavor, used for running code on desktop machines.""" + + +WIN_TOOLCHAIN_DIR = 't' + + +class DeviceDirs(object): + def __init__(self, + bin_dir, + dm_dir, + perf_data_dir, + resource_dir, + images_dir, + skp_dir, + svg_dir, + tmp_dir): + self._bin_dir = bin_dir + self._dm_dir = dm_dir + self._perf_data_dir = perf_data_dir + self._resource_dir = resource_dir + self._images_dir = images_dir + self._skp_dir = skp_dir + self._svg_dir = svg_dir + self._tmp_dir = tmp_dir + + @property + def bin_dir(self): + return self._bin_dir + + @property + def dm_dir(self): + """Where DM writes.""" + return self._dm_dir + + @property + def perf_data_dir(self): + return self._perf_data_dir + + @property + def resource_dir(self): + return self._resource_dir + + @property + def images_dir(self): + return self._images_dir + + @property + def skp_dir(self): + """Holds SKP files that are consumed by RenderSKPs and BenchPictures.""" + return self._skp_dir + + @property + def svg_dir(self): + return self._svg_dir + + @property + def tmp_dir(self): + return self._tmp_dir + + +class DefaultFlavor(object): + def __init__(self, module): + # Store a pointer to the parent recipe module (SkiaFlavorApi) so that + # FlavorUtils objects can do recipe module-like things, like run steps or + # access module-level resources. + self.module = module + + # self.m is just a shortcut so that Flavor objects can use the same + # syntax as regular recipe modules to run steps, eg: self.m.step(...) + self.m = module.m + self._chrome_path = None + self.device_dirs = DeviceDirs( + bin_dir=self.m.vars.build_dir.join('out', self.m.vars.configuration), + dm_dir=self.m.path.join(self.m.vars.swarming_out_dir, 'dm'), + perf_data_dir=self.m.path.join( + self.m.vars.swarming_out_dir, + 'perfdata', self.m.vars.builder_name, 'data'), + resource_dir=self.m.path['start_dir'].join('skia', 'resources'), + images_dir=self.m.path['start_dir'].join('skimage'), + skp_dir=self.m.path['start_dir'].join('skp'), + svg_dir=self.m.path['start_dir'].join('svg'), + tmp_dir=self.m.vars.tmp_dir) + self.host_dirs = self.device_dirs + + def device_path_join(self, *args): + """Like os.path.join(), but for paths on a connected device.""" + return self.m.path.join(*args) + + def copy_directory_contents_to_device(self, host_dir, device_dir): + """Like shutil.copytree(), but for copying to a connected device.""" + # For "normal" builders who don't have an attached device, we expect + # host_dir and device_dir to be the same. + if str(host_dir) != str(device_dir): + raise ValueError('For builders who do not have attached devices, copying ' + 'from host to device is undefined and only allowed if ' + 'host_path and device_path are the same (%s vs %s).' % ( + str(host_dir), str(device_dir))) + + def copy_directory_contents_to_host(self, device_dir, host_dir): + """Like shutil.copytree(), but for copying from a connected device.""" + # For "normal" builders who don't have an attached device, we expect + # host_dir and device_dir to be the same. + if str(host_dir) != str(device_dir): + raise ValueError('For builders who do not have attached devices, copying ' + 'from device to host is undefined and only allowed if ' + 'host_path and device_path are the same (%s vs %s).' % ( + str(host_dir), str(device_dir))) + + def copy_file_to_device(self, host_path, device_path): + """Like shutil.copyfile, but for copying to a connected device.""" + # For "normal" builders who don't have an attached device, we expect + # host_dir and device_dir to be the same. + if str(host_path) != str(device_path): + raise ValueError('For builders who do not have attached devices, copying ' + 'from host to device is undefined and only allowed if ' + 'host_path and device_path are the same (%s vs %s).' % ( + str(host_path), str(device_path))) + + def create_clean_device_dir(self, path): + """Like shutil.rmtree() + os.makedirs(), but on a connected device.""" + self.create_clean_host_dir(path) + + def create_clean_host_dir(self, path): + """Convenience function for creating a clean directory.""" + self.m.run.rmtree(path) + self.m.file.ensure_directory( + 'makedirs %s' % self.m.path.basename(path), path) + + def install(self): + """Run device-specific installation steps.""" + pass + + def cleanup_steps(self): + """Run any device-specific cleanup steps.""" + pass + def _run(self, title, cmd, infra_step=False, **kwargs): return self.m.run(self.m.step, title, cmd=cmd, infra_step=infra_step, **kwargs) diff --git a/infra/bots/recipe_modules/flavor/default_flavor.py b/infra/bots/recipe_modules/flavor/default_flavor.py deleted file mode 100644 index ec684efd1f..0000000000 --- a/infra/bots/recipe_modules/flavor/default_flavor.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2014 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -# pylint: disable=W0201 - - -"""Default flavor utils class, used for desktop builders.""" - - -import json - - -WIN_TOOLCHAIN_DIR = 't' - - -class DeviceDirs(object): - def __init__(self, - bin_dir, - dm_dir, - perf_data_dir, - resource_dir, - images_dir, - skp_dir, - svg_dir, - tmp_dir): - self._bin_dir = bin_dir - self._dm_dir = dm_dir - self._perf_data_dir = perf_data_dir - self._resource_dir = resource_dir - self._images_dir = images_dir - self._skp_dir = skp_dir - self._svg_dir = svg_dir - self._tmp_dir = tmp_dir - - @property - def bin_dir(self): - return self._bin_dir - - @property - def dm_dir(self): - """Where DM writes.""" - return self._dm_dir - - @property - def perf_data_dir(self): - return self._perf_data_dir - - @property - def resource_dir(self): - return self._resource_dir - - @property - def images_dir(self): - return self._images_dir - - @property - def skp_dir(self): - """Holds SKP files that are consumed by RenderSKPs and BenchPictures.""" - return self._skp_dir - - @property - def svg_dir(self): - return self._svg_dir - - @property - def tmp_dir(self): - return self._tmp_dir - - -class DefaultFlavorUtils(object): - """Utilities to be used by build steps. - - The methods in this class define how certain high-level functions should - work. Each build step flavor should correspond to a subclass of - DefaultFlavorUtils which may override any of these functions as appropriate - for that flavor. - - For example, the AndroidFlavorUtils will override the functions for - copying files between the host and Android device, as well as the - 'step' function, so that commands may be run through ADB. - """ - def __init__(self, module): - # Store a pointer to the parent recipe module (SkiaFlavorApi) so that - # FlavorUtils objects can do recipe module-like things, like run steps or - # access module-level resources. - self.module = module - - # self.m is just a shortcut so that FlavorUtils objects can use the same - # syntax as regular recipe modules to run steps, eg: self.m.step(...) - self.m = module.m - self._chrome_path = None - self.device_dirs = DeviceDirs( - bin_dir=self.m.vars.build_dir.join('out', self.m.vars.configuration), - dm_dir=self.m.path.join(self.m.vars.swarming_out_dir, 'dm'), - perf_data_dir=self.m.path.join( - self.m.vars.swarming_out_dir, - 'perfdata', self.m.vars.builder_name, 'data'), - resource_dir=self.m.path['start_dir'].join('skia', 'resources'), - images_dir=self.m.path['start_dir'].join('skimage'), - skp_dir=self.m.path['start_dir'].join('skp'), - svg_dir=self.m.path['start_dir'].join('svg'), - tmp_dir=self.m.vars.tmp_dir) - self.host_dirs = self.device_dirs - - def device_path_join(self, *args): - """Like os.path.join(), but for paths on a connected device.""" - return self.m.path.join(*args) - - def copy_directory_contents_to_device(self, host_dir, device_dir): - """Like shutil.copytree(), but for copying to a connected device.""" - # For "normal" builders who don't have an attached device, we expect - # host_dir and device_dir to be the same. - if str(host_dir) != str(device_dir): - raise ValueError('For builders who do not have attached devices, copying ' - 'from host to device is undefined and only allowed if ' - 'host_path and device_path are the same (%s vs %s).' % ( - str(host_dir), str(device_dir))) - - def copy_directory_contents_to_host(self, device_dir, host_dir): - """Like shutil.copytree(), but for copying from a connected device.""" - # For "normal" builders who don't have an attached device, we expect - # host_dir and device_dir to be the same. - if str(host_dir) != str(device_dir): - raise ValueError('For builders who do not have attached devices, copying ' - 'from device to host is undefined and only allowed if ' - 'host_path and device_path are the same (%s vs %s).' % ( - str(host_dir), str(device_dir))) - - def copy_file_to_device(self, host_path, device_path): - """Like shutil.copyfile, but for copying to a connected device.""" - # For "normal" builders who don't have an attached device, we expect - # host_dir and device_dir to be the same. - if str(host_path) != str(device_path): - raise ValueError('For builders who do not have attached devices, copying ' - 'from host to device is undefined and only allowed if ' - 'host_path and device_path are the same (%s vs %s).' % ( - str(host_path), str(device_path))) - - def create_clean_device_dir(self, path): - """Like shutil.rmtree() + os.makedirs(), but on a connected device.""" - self.create_clean_host_dir(path) - - def create_clean_host_dir(self, path): - """Convenience function for creating a clean directory.""" - self.m.run.rmtree(path) - self.m.file.ensure_directory( - 'makedirs %s' % self.m.path.basename(path), path) - - def install(self): - """Run device-specific installation steps.""" - pass - - def cleanup_steps(self): - """Run any device-specific cleanup steps.""" - pass diff --git a/infra/bots/recipe_modules/flavor/ios_flavor.py b/infra/bots/recipe_modules/flavor/ios.py index 3df07451e9..d78cd4ca39 100644 --- a/infra/bots/recipe_modules/flavor/ios_flavor.py +++ b/infra/bots/recipe_modules/flavor/ios.py @@ -5,14 +5,17 @@ # Disable warning about setting self.device_dirs in install(); we need to. # pylint: disable=W0201 -import default_flavor -import gn_flavor -import os -class iOSFlavorUtils(gn_flavor.GNFlavorUtils): +from . import default + + +"""iOS flavor, used for running code on iOS.""" + + +class iOSFlavor(default.DefaultFlavor): def __init__(self, m): - super(iOSFlavorUtils, self).__init__(m) - self.device_dirs = default_flavor.DeviceDirs( + super(iOSFlavor, self).__init__(m) + self.device_dirs = default.DeviceDirs( bin_dir='[unused]', dm_dir='dm', perf_data_dir='perf', diff --git a/infra/bots/recipe_modules/flavor/valgrind_flavor.py b/infra/bots/recipe_modules/flavor/valgrind.py index b54fcc36dd..774dbd461a 100644 --- a/infra/bots/recipe_modules/flavor/valgrind_flavor.py +++ b/infra/bots/recipe_modules/flavor/valgrind.py @@ -3,15 +3,15 @@ # found in the LICENSE file. -import gn_flavor +from . import default -"""Utils for running under Valgrind.""" +"""Valgrind flavor, used for running code through Valgrind.""" -class ValgrindFlavorUtils(gn_flavor.GNFlavorUtils): +class ValgrindFlavor(default.DefaultFlavor): def __init__(self, m): - super(ValgrindFlavorUtils, self).__init__(m) + super(ValgrindFlavor, self).__init__(m) self._suppressions_file = self.m.path['start_dir'].join( 'skia', 'tools', 'valgrind.supp') self._valgrind_cipd_dir = self.m.vars.slave_dir.join('valgrind') diff --git a/infra/bots/recipe_modules/infra/examples/full.py b/infra/bots/recipe_modules/infra/examples/full.py index 84b2e2c442..3db5de26cb 100644 --- a/infra/bots/recipe_modules/infra/examples/full.py +++ b/infra/bots/recipe_modules/infra/examples/full.py @@ -7,7 +7,6 @@ DEPS = [ - 'core', 'infra', 'recipe_engine/file', 'recipe_engine/path', diff --git a/infra/bots/recipes/bookmaker.py b/infra/bots/recipes/bookmaker.py index f499422516..3a568ec9bf 100644 --- a/infra/bots/recipes/bookmaker.py +++ b/infra/bots/recipes/bookmaker.py @@ -18,7 +18,7 @@ DEPS = [ 'recipe_engine/path', 'recipe_engine/properties', 'recipe_engine/step', - 'core', + 'checkout', 'infra', 'run', 'vars', @@ -39,8 +39,8 @@ def go_get_fiddlecli(api): def RunSteps(api): api.vars.setup() - checkout_root = api.core.default_checkout_root - api.core.checkout_bot_update(checkout_root=checkout_root) + checkout_root = api.checkout.default_checkout_root + api.checkout.bot_update(checkout_root=checkout_root) api.infra.go_version() go_get_fiddlecli(api) diff --git a/infra/bots/recipes/calmbench.py b/infra/bots/recipes/calmbench.py index f75cd35d72..659b3a4f02 100644 --- a/infra/bots/recipes/calmbench.py +++ b/infra/bots/recipes/calmbench.py @@ -6,7 +6,6 @@ # Recipe module for Skia Swarming calmbench. DEPS = [ - 'core', 'flavor', 'recipe_engine/context', 'recipe_engine/file', diff --git a/infra/bots/recipes/check_generated_files.py b/infra/bots/recipes/check_generated_files.py index 61a859f26a..537d7758da 100644 --- a/infra/bots/recipes/check_generated_files.py +++ b/infra/bots/recipes/check_generated_files.py @@ -14,7 +14,7 @@ DEPS = [ 'recipe_engine/python', 'recipe_engine/raw_io', 'recipe_engine/step', - 'core', + 'checkout', 'flavor', 'run', 'vars', @@ -24,8 +24,8 @@ DEPS = [ def RunSteps(api): # Checkout, compile, etc. api.vars.setup() - checkout_root = api.core.default_checkout_root - api.core.checkout_bot_update(checkout_root=checkout_root) + checkout_root = api.checkout.default_checkout_root + api.checkout.bot_update(checkout_root=checkout_root) api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) api.flavor.setup() diff --git a/infra/bots/recipes/compile.py b/infra/bots/recipes/compile.py index 96ed7a9cbc..e5e553cd16 100644 --- a/infra/bots/recipes/compile.py +++ b/infra/bots/recipes/compile.py @@ -8,7 +8,7 @@ DEPS = [ 'build', - 'core', + 'checkout', 'recipe_engine/context', 'recipe_engine/file', 'recipe_engine/json', @@ -28,12 +28,12 @@ def RunSteps(api): # Check out code. if 'NoDEPS' in api.properties['buildername']: checkout_root = api.path['start_dir'] - api.core.checkout_git(checkout_root=checkout_root) + api.checkout.git(checkout_root=checkout_root) else: - checkout_root = api.core.default_checkout_root + checkout_root = api.checkout.default_checkout_root if 'Flutter' in api.vars.builder_name: checkout_root = checkout_root.join('flutter') - api.core.checkout_bot_update(checkout_root=checkout_root) + api.checkout.bot_update(checkout_root=checkout_root) api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) diff --git a/infra/bots/recipes/ct_skps.py b/infra/bots/recipes/ct_skps.py index 5a679d30b5..9e85a2ecba 100644 --- a/infra/bots/recipes/ct_skps.py +++ b/infra/bots/recipes/ct_skps.py @@ -8,7 +8,7 @@ import math DEPS = [ 'build', - 'core', + 'checkout', 'ct', 'recipe_engine/context', 'recipe_engine/file', @@ -78,8 +78,8 @@ def RunSteps(api): api.vars.setup() checkout_root = make_path(api, '/', 'b', 'work') gclient_cache = make_path(api, '/', 'b', 'cache') - got_revision = api.core.checkout_bot_update(checkout_root=checkout_root, - gclient_cache=gclient_cache) + got_revision = api.checkout.bot_update(checkout_root=checkout_root, + gclient_cache=gclient_cache) api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) out_dir = api.vars.build_dir.join('out', api.vars.configuration) diff --git a/infra/bots/recipes/housekeeper.expected/Housekeeper-PerCommit-Trybot.json b/infra/bots/recipes/housekeeper.expected/Housekeeper-PerCommit-Trybot.json index 2863e42c9b..8d7f7912fb 100644 --- a/infra/bots/recipes/housekeeper.expected/Housekeeper-PerCommit-Trybot.json +++ b/infra/bots/recipes/housekeeper.expected/Housekeeper-PerCommit-Trybot.json @@ -124,14 +124,13 @@ "0777", "[START_DIR]/[SWARM_OUT_DIR]/perfdata/Housekeeper-PerCommit/data" ], - "cwd": "[START_DIR]/cache/work/skia", "infra_step": true, "name": "makedirs perf_dir" }, { "cmd": [ "python", - "RECIPE_MODULE[skia::core]/resources/run_binary_size_analysis.py", + "RECIPE_MODULE[skia::binary_size]/resources/run_binary_size_analysis.py", "--library", "[START_DIR]/build/out/Release/libskia.so", "--githash", diff --git a/infra/bots/recipes/housekeeper.expected/Housekeeper-PerCommit.json b/infra/bots/recipes/housekeeper.expected/Housekeeper-PerCommit.json index ccc1ff38b5..d152ff9c82 100644 --- a/infra/bots/recipes/housekeeper.expected/Housekeeper-PerCommit.json +++ b/infra/bots/recipes/housekeeper.expected/Housekeeper-PerCommit.json @@ -111,7 +111,7 @@ { "cmd": [ "python", - "RECIPE_MODULE[skia::core]/resources/generate_and_upload_doxygen.py" + "RECIPE_MODULE[skia::doxygen]/resources/generate_and_upload_doxygen.py" ], "cwd": "[START_DIR]/cache/work/skia", "env": { @@ -132,14 +132,13 @@ "0777", "[START_DIR]/[SWARM_OUT_DIR]/perfdata/Housekeeper-PerCommit/data" ], - "cwd": "[START_DIR]/cache/work/skia", "infra_step": true, "name": "makedirs perf_dir" }, { "cmd": [ "python", - "RECIPE_MODULE[skia::core]/resources/run_binary_size_analysis.py", + "RECIPE_MODULE[skia::binary_size]/resources/run_binary_size_analysis.py", "--library", "[START_DIR]/build/out/Release/libskia.so", "--githash", diff --git a/infra/bots/recipes/housekeeper.py b/infra/bots/recipes/housekeeper.py index 3cc76c26e0..e565216bd4 100644 --- a/infra/bots/recipes/housekeeper.py +++ b/infra/bots/recipes/housekeeper.py @@ -10,15 +10,14 @@ import calendar DEPS = [ - 'core', - 'depot_tools/bot_update', + 'binary_size', + 'checkout', + 'doxygen', 'flavor', 'recipe_engine/context', 'recipe_engine/file', 'recipe_engine/path', 'recipe_engine/properties', - 'recipe_engine/python', - 'recipe_engine/step', 'recipe_engine/time', 'run', 'vars', @@ -28,37 +27,24 @@ DEPS = [ def RunSteps(api): # Checkout, compile, etc. api.vars.setup() - checkout_root = api.core.default_checkout_root - got_revision = api.core.checkout_bot_update(checkout_root=checkout_root) + checkout_root = api.checkout.default_checkout_root + got_revision = api.checkout.bot_update(checkout_root=checkout_root) api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) api.flavor.setup() # TODO(borenet): Detect static initializers? - with api.context(cwd=checkout_root.join('skia')): - if not api.vars.is_trybot: - api.run( - api.step, - 'generate and upload doxygen', - cmd=['python', api.core.resource('generate_and_upload_doxygen.py')], - abort_on_failure=False) + skia_dir = checkout_root.join('skia') + if not api.vars.is_trybot: + api.doxygen.generate_and_upload(skia_dir) - now = api.time.utcnow() - ts = int(calendar.timegm(now.utctimetuple())) - filename = 'nanobench_%s_%d.json' % (got_revision, ts) - dest_dir = api.flavor.host_dirs.perf_data_dir - dest_file = dest_dir + '/' + filename - api.file.ensure_directory('makedirs perf_dir', dest_dir) - cmd = ['python', api.core.resource('run_binary_size_analysis.py'), - '--library', api.vars.skia_out.join('libskia.so'), - '--githash', api.properties['revision'], - '--dest', dest_file] - if api.vars.is_trybot: - cmd.extend(['--issue_number', str(api.properties['patch_issue'])]) - api.run( - api.step, - 'generate binary size data', - cmd=cmd) + now = api.time.utcnow() + ts = int(calendar.timegm(now.utctimetuple())) + filename = 'nanobench_%s_%d.json' % (got_revision, ts) + dest_dir = api.flavor.host_dirs.perf_data_dir + dest_file = dest_dir + '/' + filename + api.file.ensure_directory('makedirs perf_dir', dest_dir) + api.binary_size.run_analysis(skia_dir, dest_file) def GenTests(api): diff --git a/infra/bots/recipes/infra.py b/infra/bots/recipes/infra.py index 902bf98833..c299d17c47 100644 --- a/infra/bots/recipes/infra.py +++ b/infra/bots/recipes/infra.py @@ -7,21 +7,19 @@ DEPS = [ + 'checkout', + 'infra', 'recipe_engine/context', - 'recipe_engine/path', 'recipe_engine/properties', 'recipe_engine/step', - 'core', - 'infra', - 'run', 'vars', ] def RunSteps(api): api.vars.setup() - checkout_root = api.core.default_checkout_root - api.core.checkout_bot_update(checkout_root=checkout_root) + checkout_root = api.checkout.default_checkout_root + api.checkout.bot_update(checkout_root=checkout_root) api.infra.update_go_deps() # Run the infra tests. diff --git a/infra/bots/recipes/perf.py b/infra/bots/recipes/perf.py index a3524adecb..7880b48273 100644 --- a/infra/bots/recipes/perf.py +++ b/infra/bots/recipes/perf.py @@ -11,7 +11,6 @@ import os DEPS = [ - 'core', 'env', 'flavor', 'recipe_engine/file', diff --git a/infra/bots/recipes/recreate_skps.py b/infra/bots/recipes/recreate_skps.py index b41d56f3f6..80f4dcf0df 100644 --- a/infra/bots/recipes/recreate_skps.py +++ b/infra/bots/recipes/recreate_skps.py @@ -7,7 +7,7 @@ DEPS = [ - 'core', + 'checkout', 'depot_tools/gclient', 'flavor', 'infra', @@ -36,8 +36,8 @@ TEST_BUILDERS = { def RunSteps(api): # Check out Chrome. api.vars.setup() - checkout_root = api.core.default_checkout_root - api.core.checkout_bot_update(checkout_root=checkout_root) + checkout_root = api.checkout.default_checkout_root + api.checkout.bot_update(checkout_root=checkout_root) api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) api.flavor.setup() diff --git a/infra/bots/recipes/skqp_test.py b/infra/bots/recipes/skqp_test.py index 6162c3741a..56c27f7fe9 100644 --- a/infra/bots/recipes/skqp_test.py +++ b/infra/bots/recipes/skqp_test.py @@ -5,7 +5,6 @@ # Recipe module for Skia Swarming SKQP testing. DEPS = [ - 'core', 'flavor', 'recipe_engine/file', 'recipe_engine/path', diff --git a/infra/bots/recipes/test.py b/infra/bots/recipes/test.py index 33a1b983b1..8145f368da 100644 --- a/infra/bots/recipes/test.py +++ b/infra/bots/recipes/test.py @@ -7,7 +7,6 @@ DEPS = [ - 'core', 'env', 'flavor', 'recipe_engine/context', diff --git a/infra/bots/recipes/upload_calmbench_results.py b/infra/bots/recipes/upload_calmbench_results.py index 8aa6c01644..ef71575b2d 100644 --- a/infra/bots/recipes/upload_calmbench_results.py +++ b/infra/bots/recipes/upload_calmbench_results.py @@ -10,7 +10,6 @@ import calendar DEPS = [ - 'core', 'flavor', 'recipe_engine/context', 'recipe_engine/file', diff --git a/infra/bots/recipes/upload_skiaserve.py b/infra/bots/recipes/upload_skiaserve.py index cee9584c77..251802b943 100644 --- a/infra/bots/recipes/upload_skiaserve.py +++ b/infra/bots/recipes/upload_skiaserve.py @@ -7,7 +7,6 @@ DEPS = [ - 'core', 'flavor', 'gsutil', 'recipe_engine/context', |