diff options
| author |  Eric Boren <borenet@google.com> | 2018-05-18 07:36:55 -0400 |
|---|---|---|
| committer |  Skia Commit-Bot <skia-commit-bot@chromium.org> | 2018-05-18 11:58:14 +0000 |
| commit | 72f6668eb73405874155b7b03077230559ee681e (patch) | |
| tree | 4dc8de37518a9e442ac8ec15f727fd32c6b312a4 /infra/bots/recipe_modules | |
| parent | 50edafacc8ea56bdc13fd2234618fe06094eec0e (diff) | |
[recipes] Move a lot of logic out of vars module
In general, vars should only contain variables which are the same for
all tasks. Variables specific to compilation belong in the build
module (or compile recipe), and those specific to running tests belong
in the flavor module, or the individual recipe which uses them.
Bug: skia:6473
Change-Id: Ifd55a57118c5801e6f4934a6b5de9d1567415b9a
Reviewed-on: https://skia-review.googlesource.com/128545
Commit-Queue: Eric Boren <borenet@google.com>
Reviewed-by: Ben Wagner <benjaminwagner@google.com>
Diffstat (limited to 'infra/bots/recipe_modules')
30 files changed, 532 insertions, 594 deletions
diff --git a/infra/bots/recipe_modules/build/android.py b/infra/bots/recipe_modules/build/android.py index 82253bb947..ba57db89a1 100644 --- a/infra/bots/recipe_modules/build/android.py +++ b/infra/bots/recipe_modules/build/android.py @@ -6,7 +6,8 @@ import re -def compile_fn(api, out_dir): +def compile_fn(api, checkout_root, out_dir): + skia_dir = checkout_root.join('skia') compiler = api.vars.builder_cfg.get('compiler') configuration = api.vars.builder_cfg.get('configuration') extra_tokens = api.vars.extra_tokens @@ -34,7 +35,7 @@ def compile_fn(api, out_dir): 'target_cpu': quote(target_arch), } extra_cflags.append('-DDUMMY_ndk_version=%s' % - api.run.asset_version(ndk_asset)) + api.run.asset_version(ndk_asset, skia_dir)) if configuration != 'Debug': args['is_debug'] = 'false' @@ -59,11 +60,11 @@ def compile_fn(api, out_dir): gn_args = ' '.join('%s=%s' % (k,v) for (k,v) in sorted(args.iteritems())) gn = 'gn.exe' if 'Win' in os else 'gn' ninja = 'ninja.exe' if 'Win' in os else 'ninja' - gn = api.vars.skia_dir.join('bin', gn) + gn = skia_dir.join('bin', gn) - with api.context(cwd=api.vars.skia_dir): + with api.context(cwd=skia_dir): api.run(api.python, 'fetch-gn', - script=api.vars.skia_dir.join('bin', 'fetch-gn'), + script=skia_dir.join('bin', 'fetch-gn'), infra_step=True) # If this is the SkQP build, set up the environment and run the script @@ -72,7 +73,7 @@ def compile_fn(api, out_dir): api.infra.update_go_deps() output_binary = out_dir.join('run_testlab') - build_target = api.vars.skia_dir.join('infra', 'cts', 'run_testlab.go') + build_target = skia_dir.join('infra', 'cts', 'run_testlab.go') build_cmd = ['go', 'build', '-o', output_binary, build_target] with api.context(env=api.infra.go_env): api.run(api.step, 'build firebase runner', cmd=build_cmd) @@ -88,8 +89,7 @@ def compile_fn(api, out_dir): 'APK_OUTPUT_DIR': out_dir, } - mk_universal = api.vars.skia_dir.join('tools', 'skqp', - 'make_universal_apk') + mk_universal = skia_dir.join('tools', 'skqp', 'make_universal_apk') with api.context(env=env): api.run(api.step, 'make_universal', cmd=[mk_universal]) else: @@ -99,6 +99,4 @@ def compile_fn(api, out_dir): def copy_extra_build_products(api, src, dst): - if 'SKQP' in api.vars.extra_tokens: - wlist = api.vars.skia_dir.join('infra','cts', 'whitelist_devices.json') - api.file.copy('copy whitelist', wlist, dst) + pass diff --git a/infra/bots/recipe_modules/build/api.py b/infra/bots/recipe_modules/build/api.py index 545f94f288..286190b94a 100644 --- a/infra/bots/recipe_modules/build/api.py +++ b/infra/bots/recipe_modules/build/api.py @@ -36,13 +36,11 @@ class BuildApi(recipe_api.RecipeApi): self.copy_fn = default.copy_extra_build_products super(BuildApi, self).__init__(*args, **kwargs) - def __call__(self): + def __call__(self, checkout_root, out_dir): """Compile the code.""" - out_dir = self.m.vars.skia_out.join(self.m.vars.configuration) - self.compile_fn(self.m, out_dir) + self.compile_fn(self.m, checkout_root, out_dir) - def copy_build_products(self, dst): + def copy_build_products(self, out_dir, dst): """Copy whitelisted build products to dst.""" - src = self.m.vars.skia_out.join(self.m.vars.configuration) - util.copy_whitelisted_build_products(self.m, src, dst) - self.copy_fn(self.m, src, dst) + util.copy_whitelisted_build_products(self.m, out_dir, dst) + self.copy_fn(self.m, out_dir, dst) diff --git a/infra/bots/recipe_modules/build/chromebook.py b/infra/bots/recipe_modules/build/chromebook.py index 0ab624e2bc..f7af5552d3 100644 --- a/infra/bots/recipe_modules/build/chromebook.py +++ b/infra/bots/recipe_modules/build/chromebook.py @@ -3,7 +3,8 @@ # found in the LICENSE file. -def compile_fn(api, out_dir): +def compile_fn(api, checkout_root, out_dir): + skia_dir = checkout_root.join('skia') configuration = api.vars.builder_cfg.get('configuration') os = api.vars.builder_cfg.get('os') target_arch = api.vars.builder_cfg.get('target_arch') @@ -74,7 +75,7 @@ def compile_fn(api, out_dir): 'skia_use_egl': 'true', } extra_cflags.append('-DDUMMY_clang_linux_version=%s' % - api.run.asset_version('clang_linux')) + api.run.asset_version('clang_linux', skia_dir)) if configuration != 'Debug': args['is_debug'] = 'false' @@ -86,11 +87,11 @@ def compile_fn(api, out_dir): gn = 'gn.exe' if 'Win' in os else 'gn' ninja = 'ninja.exe' if 'Win' in os else 'ninja' - gn = api.vars.skia_dir.join('bin', gn) + gn = skia_dir.join('bin', gn) - with api.context(cwd=api.vars.skia_dir, env=env): + with api.context(cwd=skia_dir, env=env): api.run(api.python, 'fetch-gn', - script=api.vars.skia_dir.join('bin', 'fetch-gn'), + script=skia_dir.join('bin', 'fetch-gn'), infra_step=True) api.run(api.step, 'gn gen', cmd=[gn, 'gen', out_dir, '--args=' + gn_args]) api.run(api.step, 'ninja', diff --git a/infra/bots/recipe_modules/build/chromecast.py b/infra/bots/recipe_modules/build/chromecast.py index b1631aa6c0..d0e5b2f4ef 100644 --- a/infra/bots/recipe_modules/build/chromecast.py +++ b/infra/bots/recipe_modules/build/chromecast.py @@ -3,7 +3,8 @@ # found in the LICENSE file. -def compile_fn(api, out_dir): +def compile_fn(api, checkout_root, out_dir): + skia_dir = checkout_root.join('skia') configuration = api.vars.builder_cfg.get('configuration') os = api.vars.builder_cfg.get('os') target_arch = api.vars.builder_cfg.get('target_arch') @@ -21,7 +22,7 @@ def compile_fn(api, out_dir): # Makes the binary small enough to fit on the small disk. '-g0', ('-DDUMMY_cast_toolchain_version=%s' % - api.run.asset_version('cast_toolchain')), + api.run.asset_version('cast_toolchain', skia_dir)), ] extra_ldflags = [ @@ -55,11 +56,11 @@ def compile_fn(api, out_dir): gn = 'gn.exe' if 'Win' in os else 'gn' ninja = 'ninja.exe' if 'Win' in os else 'ninja' - gn = api.vars.skia_dir.join('bin', gn) + gn = skia_dir.join('bin', gn) - with api.context(cwd=api.vars.skia_dir): + with api.context(cwd=skia_dir): api.run(api.python, 'fetch-gn', - script=api.vars.skia_dir.join('bin', 'fetch-gn'), + script=skia_dir.join('bin', 'fetch-gn'), infra_step=True) api.run(api.step, 'gn gen', cmd=[gn, 'gen', out_dir, '--args=' + gn_args]) api.run(api.step, 'ninja', diff --git a/infra/bots/recipe_modules/build/default.py b/infra/bots/recipe_modules/build/default.py index a7ee9801d6..da96dc9e60 100644 --- a/infra/bots/recipe_modules/build/default.py +++ b/infra/bots/recipe_modules/build/default.py @@ -6,12 +6,12 @@ from . import util -def build_command_buffer(api): +def build_command_buffer(api, chrome_dir, skia_dir, out): api.run(api.python, 'build command_buffer', - script=api.vars.skia_dir.join('tools', 'build_command_buffer.py'), + script=skia_dir.join('tools', 'build_command_buffer.py'), args=[ - '--chrome-dir', api.vars.checkout_root, - '--output-dir', api.vars.skia_out.join(api.vars.configuration), + '--chrome-dir', chrome_dir, + '--output-dir', out, '--no-sync', '--no-hooks', '--make-output-dir']) @@ -40,7 +40,8 @@ def compile_swiftshader(api, swiftshader_root, cc, cxx, out): cmd=['ninja', '-C', out, 'libEGL.so', 'libGLESv2.so']) -def compile_fn(api, out_dir): +def compile_fn(api, checkout_root, out_dir): + skia_dir = checkout_root.join('skia') compiler = api.vars.builder_cfg.get('compiler', '') configuration = api.vars.builder_cfg.get('configuration', '') extra_tokens = api.vars.extra_tokens @@ -69,7 +70,7 @@ def compile_fn(api, out_dir): extra_ldflags.append('-B%s/bin' % clang_linux) extra_ldflags.append('-fuse-ld=lld') extra_cflags.append('-DDUMMY_clang_linux_version=%s' % - api.run.asset_version('clang_linux')) + api.run.asset_version('clang_linux', skia_dir)) if os == 'Ubuntu14': extra_ldflags.extend(['-static-libstdc++', '-static-libgcc']) @@ -89,7 +90,7 @@ def compile_fn(api, out_dir): '-Wno-format-truncation', '-Wno-uninitialized', ('-DDUMMY_mips64el_toolchain_linux_version=%s' % - api.run.asset_version('mips64el_toolchain_linux')) + api.run.asset_version('mips64el_toolchain_linux', skia_dir)) ]) if configuration == 'Release': # This warning is only triggered when fuzz_canvas is inlined. @@ -106,7 +107,7 @@ def compile_fn(api, out_dir): cxx = emscripten_sdk + '/emscripten/incoming/em++' extra_cflags.append('-Wno-unknown-warning-option') extra_cflags.append('-DDUMMY_emscripten_sdk_version=%s' % - api.run.asset_version('emscripten_sdk')) + api.run.asset_version('emscripten_sdk', skia_dir)) if 'Coverage' in extra_tokens: # See https://clang.llvm.org/docs/SourceBasedCodeCoverage.html for # more info on using llvm to gather coverage information. @@ -139,8 +140,7 @@ def compile_fn(api, out_dir): if 'ANGLE' in extra_tokens: args['skia_use_angle'] = 'true' if 'SwiftShader' in extra_tokens: - swiftshader_root = api.vars.skia_dir.join('third_party', 'externals', - 'swiftshader') + swiftshader_root = skia_dir.join('third_party', 'externals', 'swiftshader') swiftshader_out = out_dir.join('swiftshader_out') compile_swiftshader(api, swiftshader_root, cc, cxx, swiftshader_out) args['skia_use_egl'] = 'true' @@ -154,7 +154,8 @@ def compile_fn(api, out_dir): '-L%s' % swiftshader_out, ]) if 'CommandBuffer' in extra_tokens: - api.run.run_once(build_command_buffer, api) + chrome_dir = checkout_root + api.run.run_once(build_command_buffer, api, chrome_dir, skia_dir, out_dir) if 'MSAN' in extra_tokens: args['skia_enable_gpu'] = 'false' args['skia_use_fontconfig'] = 'false' @@ -215,7 +216,7 @@ def compile_fn(api, out_dir): if compiler == 'Clang' and 'Win' in os: args['clang_win'] = '"%s"' % api.vars.slave_dir.join('clang_win') extra_cflags.append('-DDUMMY_clang_win_version=%s' % - api.run.asset_version('clang_win')) + api.run.asset_version('clang_win', skia_dir)) if target_arch == 'wasm': args.update({ 'skia_use_freetype': 'false', @@ -253,18 +254,18 @@ def compile_fn(api, out_dir): gn = 'gn.exe' if 'Win' in os else 'gn' ninja = 'ninja.exe' if 'Win' in os else 'ninja' - gn = api.vars.skia_dir.join('bin', gn) + gn = skia_dir.join('bin', gn) - with api.context(cwd=api.vars.skia_dir): + with api.context(cwd=skia_dir): api.run(api.python, 'fetch-gn', - script=api.vars.skia_dir.join('bin', 'fetch-gn'), + script=skia_dir.join('bin', 'fetch-gn'), infra_step=True) if 'CheckGeneratedFiles' in extra_tokens: - env['PATH'] = '%s:%%(PATH)s' % api.vars.skia_dir.join('bin') + env['PATH'] = '%s:%%(PATH)s' % skia_dir.join('bin') api.run(api.python, 'fetch-clang-format', - script=api.vars.skia_dir.join('bin', 'fetch-clang-format'), + script=skia_dir.join('bin', 'fetch-clang-format'), infra_step=True) if target_arch == 'wasm': fastcomp = emscripten_sdk + '/clang/fastcomp/build_incoming_64/bin' diff --git a/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-Clang-universal-devrel-Android_SKQP.json b/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-Clang-universal-devrel-Android_SKQP.json index 762dc757d5..f98a77e515 100644 --- a/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-Clang-universal-devrel-Android_SKQP.json +++ b/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-Clang-universal-devrel-Android_SKQP.json @@ -142,20 +142,6 @@ ] }, { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "[START_DIR]/cache/work/skia/infra/cts/whitelist_devices.json", - "[START_DIR]/[SWARM_OUT_DIR]/out/devrel" - ], - "infra_step": true, - "name": "copy whitelist" - }, - { "name": "$result", "recipe_result": null, "status_code": 0 diff --git a/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json b/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json index 3be7441907..90d0df0f2d 100644 --- a/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json +++ b/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json @@ -30,7 +30,7 @@ "cmd": [ "ninja", "-C", - "out/android_release", + "[START_DIR]/cache/work/flutter/src/out/android_release", "-j100" ], "cwd": "[START_DIR]/cache/work/flutter/src", @@ -45,7 +45,7 @@ "python", "-u", "import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n", - "[START_DIR]/cache/work/flutter/src/third_party/skia/out/Build-Debian9-GCC-x86_64-Release-Flutter_Android/Release", + "[START_DIR]/cache/work/flutter/src/out/android_release", "[START_DIR]/[SWARM_OUT_DIR]/out/Release" ], "infra_step": true, diff --git a/infra/bots/recipe_modules/build/examples/full.py b/infra/bots/recipe_modules/build/examples/full.py index 8b801410f5..aca20a6195 100644 --- a/infra/bots/recipe_modules/build/examples/full.py +++ b/infra/bots/recipe_modules/build/examples/full.py @@ -5,6 +5,7 @@ DEPS = [ 'build', + 'recipe_engine/path', 'recipe_engine/properties', 'recipe_engine/raw_io', 'run', @@ -15,10 +16,24 @@ DEPS = [ def RunSteps(api): api.vars.setup() - api.build() - api.build.copy_build_products( - api.vars.swarming_out_dir.join( - 'out', api.vars.configuration)) + # Hackery to avoid changing expectations. + checkout_root = api.vars.cache_dir.join('work') + out_dir = checkout_root.join( + 'skia', 'out', api.vars.builder_name, api.vars.configuration) + if ('CheckGeneratedFiles' in api.vars.builder_name or + '-CT_' in api.vars.builder_name): + out_dir = api.vars.build_dir.join('out', api.vars.configuration) + elif 'NoDEPS' in api.vars.builder_name: + checkout_root = api.path['start_dir'] + out_dir = checkout_root.join( + 'skia', 'out', api.vars.builder_name, api.vars.configuration) + elif 'Flutter' in api.vars.builder_name: + checkout_root = checkout_root.join('flutter') + out_dir = checkout_root.join('src', 'out', 'android_release') + + api.build(checkout_root=checkout_root, out_dir=out_dir) + dst = api.vars.swarming_out_dir.join('out', api.vars.configuration) + api.build.copy_build_products(out_dir=out_dir, dst=dst) api.run.check_failure() diff --git a/infra/bots/recipe_modules/build/flutter.py b/infra/bots/recipe_modules/build/flutter.py index fee92c1621..e1fc52e25f 100644 --- a/infra/bots/recipe_modules/build/flutter.py +++ b/infra/bots/recipe_modules/build/flutter.py @@ -3,11 +3,10 @@ # found in the LICENSE file. -def compile_fn(api, _): - flutter_dir = api.vars.checkout_root.join('src') +def compile_fn(api, checkout_root, out_dir): + flutter_dir = checkout_root.join('src') configuration = api.vars.builder_cfg.get('configuration').lower() extra_tokens = api.vars.extra_tokens - out_dir = configuration with api.context(cwd=flutter_dir): # Setup GN args. @@ -16,10 +15,9 @@ def compile_fn(api, _): ] if 'Android' in extra_tokens: gn_args.append('--android') - out_dir = 'android_' + out_dir # Delete out_dir so that we start from a clean slate. See skbug/6310. - api.run.rmtree(flutter_dir.join('out', out_dir)) + api.run.rmtree(out_dir) # Run GN. api.run( @@ -31,7 +29,7 @@ def compile_fn(api, _): api.run( api.step, 'build_flutter', - cmd=['ninja', '-C', 'out/' + out_dir, '-j100']) + cmd=['ninja', '-C', out_dir, '-j100']) def copy_extra_build_products(api, src, dst): diff --git a/infra/bots/recipe_modules/core/api.py b/infra/bots/recipe_modules/core/api.py index f7b5113255..f943b865b2 100644 --- a/infra/bots/recipe_modules/core/api.py +++ b/infra/bots/recipe_modules/core/api.py @@ -17,39 +17,58 @@ from recipe_engine import config_types class SkiaApi(recipe_api.RecipeApi): + @property + def default_checkout_root(self): + """The default location for cached persistent checkouts.""" + return self.m.vars.cache_dir.join('work') + def patch_ref(self, issue, patchset): """Build a ref for the given issue and patchset.""" return 'refs/changes/%s/%s/%s' % (issue[-2:], issue, patchset) - def checkout_git(self): + def checkout_git(self, checkout_root): """Run the steps to perform a pure-git checkout without DEPS.""" + skia_dir = checkout_root.join('skia') self.m.git.checkout( - self.m.properties['repository'], dir_path=self.m.vars.skia_dir, + self.m.properties['repository'], dir_path=skia_dir, ref=self.m.properties['revision'], submodules=False) if self.m.vars.is_trybot: ref = self.patch_ref(str(self.m.vars.issue), str(self.m.vars.patchset)) self.m.git('fetch', 'origin', ref) self.m.git('checkout', 'FETCH_HEAD') self.m.git('rebase', self.m.properties['revision']) + return self.m.properties['revision'] - def checkout_bot_update(self): + def checkout_bot_update(self, checkout_root, gclient_cache=None): """Run the steps to obtain a checkout using bot_update.""" + if not gclient_cache: + gclient_cache = self.m.vars.cache_dir.join('git') + cfg_kwargs = {} is_parent_revision = 'ParentRevision' in self.m.vars.extra_tokens # Use a persistent gclient cache for Swarming. - cfg_kwargs['CACHE_DIR'] = self.m.vars.gclient_cache + cfg_kwargs['CACHE_DIR'] = gclient_cache # Create the checkout path if necessary. - if not self.m.path.exists(self.m.vars.checkout_root): - self.m.file.ensure_directory('makedirs checkout_path', - self.m.vars.checkout_root) + if not self.m.path.exists(checkout_root): + self.m.file.ensure_directory('makedirs checkout_path', checkout_root) # Initial cleanup. gclient_cfg = self.m.gclient.make_config(**cfg_kwargs) + # Some bots also require a checkout of chromium. + need_chromium_checkout = False + gclient_env = {'DEPOT_TOOLS_UPDATE': '0'} + if 'CommandBuffer' in self.m.properties['buildername']: + need_chromium_checkout = True + gclient_env['GYP_CHROMIUM_NO_ACTION'] = '0' + if 'RecreateSKPs' in self.m.properties['buildername']: + need_chromium_checkout = True + gclient_env['CPPFLAGS'] = '-DSK_ALLOW_CROSSPROCESS_PICTUREIMAGEFILTERS=1' + # Add chromium first because of skbug.com/7917. - if self.m.vars.need_chromium_checkout: + if need_chromium_checkout: chromium = gclient_cfg.solutions.add() chromium.name = 'src' chromium.managed = False @@ -57,14 +76,16 @@ class SkiaApi(recipe_api.RecipeApi): chromium.revision = 'origin/master' main_repo = self.m.properties['repository'] - if self.m.vars.need_flutter_checkout: + + need_flutter_checkout = 'Flutter' in self.m.properties['buildername'] + if need_flutter_checkout: main_repo = 'https://github.com/flutter/engine.git' main_name = self.m.path.basename(main_repo) if main_name.endswith('.git'): main_name = main_name[:-len('.git')] # Special case for flutter because it seems to need a very specific # directory structure to successfully build. - if self.m.vars.need_flutter_checkout and main_name == 'engine': + if need_flutter_checkout and main_name == 'engine': main_name = 'src/flutter' main = gclient_cfg.solutions.add() main.name = main_name @@ -81,7 +102,7 @@ class SkiaApi(recipe_api.RecipeApi): if patch_root.endswith('.git'): patch_root = patch_root[:-4] - if self.m.vars.need_flutter_checkout: + if need_flutter_checkout: # Skia is a DEP of Flutter; the 'revision' property is a Skia revision, # and any patch should be applied to Skia, not Flutter. main.revision = 'origin/master' @@ -99,7 +120,7 @@ class SkiaApi(recipe_api.RecipeApi): # TODO(rmistry): Remove the below block after there is a solution for # crbug.com/616443 - entries_file = self.m.vars.checkout_root.join('.gclient_entries') + entries_file = checkout_root.join('.gclient_entries') if self.m.path.exists(entries_file) or self._test_data.enabled: self.m.file.remove('remove %s' % entries_file, entries_file) @@ -118,7 +139,7 @@ class SkiaApi(recipe_api.RecipeApi): main.revision = main.revision + '^' self.m.gclient.c = gclient_cfg - with self.m.context(cwd=self.m.vars.checkout_root): + with self.m.context(cwd=checkout_root): update_step = self.m.bot_update.ensure_checkout( patch_root=patch_root, # The logic in ensure_checkout for this arg is fairly naive, so if @@ -128,10 +149,7 @@ class SkiaApi(recipe_api.RecipeApi): patch=not (self.m.vars.is_trybot and is_parent_revision) ) - self.m.vars.got_revision = ( - update_step.presentation.properties['got_revision']) - - if self.m.vars.need_chromium_checkout or self.m.vars.need_flutter_checkout: - with self.m.context(cwd=self.m.vars.checkout_root, - env=self.m.vars.gclient_env): + if need_chromium_checkout or need_flutter_checkout: + with self.m.context(cwd=checkout_root, env=gclient_env): self.m.gclient.runhooks() + return update_step.presentation.properties['got_revision'] diff --git a/infra/bots/recipe_modules/core/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json b/infra/bots/recipe_modules/core/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json new file mode 100644 index 0000000000..a066ac5a97 --- /dev/null +++ b/infra/bots/recipe_modules/core/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json @@ -0,0 +1,147 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/cache/work/flutter" + ], + "infra_step": true, + "name": "makedirs checkout_path" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "remove", + "[START_DIR]/cache/work/flutter/.gclient_entries" + ], + "infra_step": true, + "name": "remove [START_DIR]/cache/work/flutter/.gclient_entries" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec-path", + "cache_dir = '[START_DIR]/cache/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': True, 'name': 'src/flutter', 'url': 'https://github.com/flutter/engine.git'}]\ntarget_os = ['android']", + "--patch_root", + "src/third_party/skia", + "--revision_mapping_file", + "{\"got_flutter_revision\": \"src/flutter\", \"got_revision\": \"src/third_party/skia\"}", + "--git-cache-dir", + "[START_DIR]/cache/git", + "--cleanup-dir", + "[CLEANUP]/bot_update", + "--output_json", + "/path/to/tmp/json", + "--revision", + "src/flutter@origin/master", + "--revision", + "src/third_party/skia@abc123" + ], + "cwd": "[START_DIR]/cache/work/flutter", + "env_prefixes": { + "PATH": [ + "RECIPE_PACKAGE_REPO[depot_tools]" + ] + }, + "infra_step": true, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src/flutter\": \"origin/master\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"src/third_party/skia\": \"abc123\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src/flutter\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/src/flutter.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9221bca00ddbd888260084def81f09543281b952\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"src/third_party/skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/src/third_party/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"d9c4a4d173a97ef2832b65636b4200bb93ea8ee1\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src/third_party/skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_flutter_revision\": \"9221bca00ddbd888260084def81f09543281b952\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_flutter_revision_cp\": \"refs/heads/master@{#84512}\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"d9c4a4d173a97ef2832b65636b4200bb93ea8ee1\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#143121}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"src/flutter\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src/flutter\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/src/flutter.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9221bca00ddbd888260084def81f09543281b952\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"src/third_party/skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/src/third_party/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"d9c4a4d173a97ef2832b65636b4200bb93ea8ee1\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_flutter_revision@\"9221bca00ddbd888260084def81f09543281b952\"@@@", + "@@@SET_BUILD_PROPERTY@got_flutter_revision_cp@\"refs/heads/master@{#84512}\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"d9c4a4d173a97ef2832b65636b4200bb93ea8ee1\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#143121}\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py", + "runhooks" + ], + "cwd": "[START_DIR]/cache/work/flutter", + "env": { + "DEPOT_TOOLS_UPDATE": "0", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "gclient runhooks" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/tmp" + ], + "infra_step": true, + "name": "makedirs tmp_dir" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/core/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json b/infra/bots/recipe_modules/core/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json new file mode 100644 index 0000000000..d9db2bbf14 --- /dev/null +++ b/infra/bots/recipe_modules/core/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json @@ -0,0 +1,134 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/cache/work" + ], + "infra_step": true, + "name": "makedirs checkout_path" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "remove", + "[START_DIR]/cache/work/.gclient_entries" + ], + "infra_step": true, + "name": "remove [START_DIR]/cache/work/.gclient_entries" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec-path", + "cache_dir = '[START_DIR]/cache/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'src', 'url': 'https://chromium.googlesource.com/chromium/src.git'}, {'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]", + "--patch_root", + "skia", + "--revision_mapping_file", + "{\"got_revision\": \"skia\"}", + "--git-cache-dir", + "[START_DIR]/cache/git", + "--cleanup-dir", + "[CLEANUP]/bot_update", + "--output_json", + "/path/to/tmp/json", + "--revision", + "skia@abc123", + "--revision", + "src@origin/master" + ], + "cwd": "[START_DIR]/cache/work", + "env_prefixes": { + "PATH": [ + "RECIPE_PACKAGE_REPO[depot_tools]" + ] + }, + "infra_step": true, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"src\": \"origin/master\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"src\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py", + "runhooks" + ], + "cwd": "[START_DIR]/cache/work", + "env": { + "DEPOT_TOOLS_UPDATE": "0", + "GYP_CHROMIUM_NO_ACTION": "0", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "gclient runhooks" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/tmp" + ], + "infra_step": true, + "name": "makedirs tmp_dir" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/core/examples/full.py b/infra/bots/recipe_modules/core/examples/full.py index f3cfb29ca7..ef12119015 100644 --- a/infra/bots/recipe_modules/core/examples/full.py +++ b/infra/bots/recipe_modules/core/examples/full.py @@ -15,13 +15,18 @@ DEPS = [ def RunSteps(api): api.vars.setup() + bot_update = True if 'NoDEPS' in api.properties['buildername']: bot_update = False + if bot_update: - api.core.checkout_bot_update() + checkout_root = api.core.default_checkout_root + if 'Flutter' in api.vars.builder_name: + checkout_root = checkout_root.join('flutter') + api.core.checkout_bot_update(checkout_root=checkout_root) else: - api.core.checkout_git() + api.core.checkout_git(checkout_root=api.path['start_dir']) api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) @@ -139,3 +144,23 @@ def GenTests(api): api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt') ) ) + + buildername = 'Build-Mac-Clang-x86_64-Debug-CommandBuffer' + yield ( + api.test(buildername) + + api.properties(buildername=buildername, + repository='https://skia.googlesource.com/skia.git', + revision='abc123', + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + ) + + buildername = 'Build-Debian9-GCC-x86_64-Release-Flutter_Android' + yield ( + api.test(buildername) + + api.properties(buildername=buildername, + repository='https://skia.googlesource.com/skia.git', + revision='abc123', + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + ) diff --git a/infra/bots/recipe_modules/flavor/api.py b/infra/bots/recipe_modules/flavor/api.py index f432066e9a..c5a1be8030 100644 --- a/infra/bots/recipe_modules/flavor/api.py +++ b/infra/bots/recipe_modules/flavor/api.py @@ -64,6 +64,9 @@ class SkiaFlavorApi(recipe_api.RecipeApi): def setup(self): self._f = self.get_flavor(self.m.vars) + self.device_dirs = self._f.device_dirs + self.host_dirs = self._f.host_dirs + self._skia_dir = self.m.path['start_dir'].join('skia') def step(self, name, cmd, **kwargs): return self._f.step(name, cmd, **kwargs) @@ -97,12 +100,11 @@ class SkiaFlavorApi(recipe_api.RecipeApi): def install(self, skps=False, images=False, svgs=False, resources=False): self._f.install() - self.device_dirs = self._f.device_dirs # TODO(borenet): Only copy files which have changed. if resources: self.copy_directory_contents_to_device( - self.m.vars.resource_dir, + self.m.path['start_dir'].join('skia', 'resources'), self.device_dirs.resource_dir) if skps: @@ -138,7 +140,7 @@ class SkiaFlavorApi(recipe_api.RecipeApi): def _copy_images(self): """Download and copy test images if needed.""" - version = self.m.run.asset_version('skimage') + version = self.m.run.asset_version('skimage', self._skia_dir) self.m.run.writefile( self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_SK_IMAGE), version) @@ -146,13 +148,13 @@ class SkiaFlavorApi(recipe_api.RecipeApi): version, VERSION_FILE_SK_IMAGE, self.m.vars.tmp_dir, - self.m.vars.images_dir, + self.host_dirs.images_dir, self.device_dirs.images_dir) return version def _copy_skps(self): """Download and copy the SKPs if needed.""" - version = self.m.run.asset_version('skp') + version = self.m.run.asset_version('skp', self._skia_dir) self.m.run.writefile( self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_SKP), version) @@ -160,13 +162,13 @@ class SkiaFlavorApi(recipe_api.RecipeApi): version, VERSION_FILE_SKP, self.m.vars.tmp_dir, - self.m.vars.local_skp_dir, + self.host_dirs.skp_dir, self.device_dirs.skp_dir) return version def _copy_svgs(self): """Download and copy the SVGs if needed.""" - version = self.m.run.asset_version('svg') + version = self.m.run.asset_version('svg', self._skia_dir) self.m.run.writefile( self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_SVG), version) @@ -174,6 +176,6 @@ class SkiaFlavorApi(recipe_api.RecipeApi): version, VERSION_FILE_SVG, self.m.vars.tmp_dir, - self.m.vars.local_svg_dir, + self.host_dirs.svg_dir, self.device_dirs.svg_dir) return version diff --git a/infra/bots/recipe_modules/flavor/default_flavor.py b/infra/bots/recipe_modules/flavor/default_flavor.py index 7ce61775ac..ec684efd1f 100644 --- a/infra/bots/recipe_modules/flavor/default_flavor.py +++ b/infra/bots/recipe_modules/flavor/default_flavor.py @@ -17,6 +17,7 @@ WIN_TOOLCHAIN_DIR = 't' class DeviceDirs(object): def __init__(self, + bin_dir, dm_dir, perf_data_dir, resource_dir, @@ -24,6 +25,7 @@ class DeviceDirs(object): skp_dir, svg_dir, tmp_dir): + self._bin_dir = bin_dir self._dm_dir = dm_dir self._perf_data_dir = perf_data_dir self._resource_dir = resource_dir @@ -33,6 +35,10 @@ class DeviceDirs(object): self._tmp_dir = tmp_dir @property + def bin_dir(self): + return self._bin_dir + + @property def dm_dir(self): """Where DM writes.""" return self._dm_dir @@ -85,11 +91,18 @@ class DefaultFlavorUtils(object): # syntax as regular recipe modules to run steps, eg: self.m.step(...) self.m = module.m self._chrome_path = None - self._win_toolchain_dir = self.m.vars.slave_dir.join(WIN_TOOLCHAIN_DIR) - win_toolchain_asset_path = self.m.vars.infrabots_dir.join( - 'assets', 'win_toolchain', 'VERSION') - if not self.m.path.exists(win_toolchain_asset_path): - self._win_toolchain_dir = self.m.vars.slave_dir + self.device_dirs = DeviceDirs( + bin_dir=self.m.vars.build_dir.join('out', self.m.vars.configuration), + dm_dir=self.m.path.join(self.m.vars.swarming_out_dir, 'dm'), + perf_data_dir=self.m.path.join( + self.m.vars.swarming_out_dir, + 'perfdata', self.m.vars.builder_name, 'data'), + resource_dir=self.m.path['start_dir'].join('skia', 'resources'), + images_dir=self.m.path['start_dir'].join('skimage'), + skp_dir=self.m.path['start_dir'].join('skp'), + svg_dir=self.m.path['start_dir'].join('svg'), + tmp_dir=self.m.vars.tmp_dir) + self.host_dirs = self.device_dirs def device_path_join(self, *args): """Like os.path.join(), but for paths on a connected device.""" @@ -137,14 +150,7 @@ class DefaultFlavorUtils(object): def install(self): """Run device-specific installation steps.""" - self.device_dirs = DeviceDirs( - dm_dir=self.m.vars.dm_dir, - perf_data_dir=self.m.vars.perf_data_dir, - resource_dir=self.m.vars.resource_dir, - images_dir=self.m.vars.images_dir, - skp_dir=self.m.vars.local_skp_dir, - svg_dir=self.m.vars.local_svg_dir, - tmp_dir=self.m.vars.tmp_dir) + pass def cleanup_steps(self): """Run any device-specific cleanup steps.""" diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/Perf-Ubuntu14-Clang-GCE-CPU-AVX2-x86_64-Release-All-CT_BENCH_1k_SKPs.json b/infra/bots/recipe_modules/flavor/examples/full.expected/Perf-Ubuntu14-Clang-GCE-CPU-AVX2-x86_64-Release-All-CT_BENCH_1k_SKPs.json deleted file mode 100644 index bd783229ba..0000000000 --- a/infra/bots/recipe_modules/flavor/examples/full.expected/Perf-Ubuntu14-Clang-GCE-CPU-AVX2-x86_64-Release-All-CT_BENCH_1k_SKPs.json +++ /dev/null @@ -1,164 +0,0 @@ -[ - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "rmtree", - "results_dir" - ], - "infra_step": true, - "name": "rmtree results_dir" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "ensure-directory", - "--mode", - "0777", - "results_dir" - ], - "infra_step": true, - "name": "makedirs results_dir" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "rmtree", - "device_results_dir" - ], - "infra_step": true, - "name": "rmtree device_results_dir" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "ensure-directory", - "--mode", - "0777", - "device_results_dir" - ], - "infra_step": true, - "name": "makedirs device_results_dir" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "[START_DIR]/cache/work/skia/infra/bots/assets/skp/VERSION", - "/path/to/tmp/" - ], - "infra_step": true, - "name": "Get skp VERSION" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "42", - "[START_DIR]/tmp/SKP_VERSION" - ], - "infra_step": true, - "name": "write SKP_VERSION" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "[START_DIR]/cache/work/skia/infra/bots/assets/skimage/VERSION", - "/path/to/tmp/" - ], - "infra_step": true, - "name": "Get skimage VERSION" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "42", - "[START_DIR]/tmp/SK_IMAGE_VERSION" - ], - "infra_step": true, - "name": "write SK_IMAGE_VERSION" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "[START_DIR]/cache/work/skia/infra/bots/assets/svg/VERSION", - "/path/to/tmp/" - ], - "infra_step": true, - "name": "Get svg VERSION" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "42", - "[START_DIR]/tmp/SVG_VERSION" - ], - "infra_step": true, - "name": "write SVG_VERSION" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[skia::flavor]/resources/symbolize_stack_trace.py", - "[START_DIR]", - "catchsegv", - "[START_DIR]/build/out/Release/nanobench", - "--some-flag" - ], - "cwd": "[START_DIR]/cache/work/skia", - "env": { - "CHROME_HEADLESS": "1", - "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" - }, - "name": "symbolized nanobench" - }, - { - "name": "$result", - "recipe_result": null, - "status_code": 0 - } -]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/flavor/examples/full.py b/infra/bots/recipe_modules/flavor/examples/full.py index 5d30a4f29b..bce549aeb3 100644 --- a/infra/bots/recipe_modules/flavor/examples/full.py +++ b/infra/bots/recipe_modules/flavor/examples/full.py @@ -43,11 +43,12 @@ def RunSteps(api): if 'Test' in api.properties['buildername']: api.flavor.step('dm', ['dm', '--some-flag']) api.flavor.copy_directory_contents_to_host( - api.flavor.device_dirs.dm_dir, api.vars.dm_dir) + api.flavor.device_dirs.dm_dir, api.flavor.host_dirs.dm_dir) elif 'Perf' in api.properties['buildername']: api.flavor.step('nanobench', ['nanobench', '--some-flag']) api.flavor.copy_directory_contents_to_host( - api.flavor.device_dirs.perf_data_dir, api.vars.perf_data_dir) + api.flavor.device_dirs.perf_data_dir, + api.flavor.host_dirs.perf_data_dir) finally: api.flavor.cleanup_steps() api.run.check_failure() @@ -62,7 +63,6 @@ TEST_BUILDERS = [ 'Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release-All', 'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-MSAN', 'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-ASAN', - 'Perf-Ubuntu14-Clang-GCE-CPU-AVX2-x86_64-Release-All-CT_BENCH_1k_SKPs', 'Test-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-Release-All-Android', 'Test-Android-Clang-GalaxyS7_G930FD-GPU-MaliT880-arm64-Debug-All-Android', 'Test-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Debug-All-Android', diff --git a/infra/bots/recipe_modules/flavor/gn_android_flavor.py b/infra/bots/recipe_modules/flavor/gn_android_flavor.py index 3f83c6d226..8348053407 100644 --- a/infra/bots/recipe_modules/flavor/gn_android_flavor.py +++ b/infra/bots/recipe_modules/flavor/gn_android_flavor.py @@ -22,14 +22,17 @@ class GNAndroidFlavorUtils(default_flavor.DefaultFlavorUtils): self.ADB_PUB_KEY = ('/home/chrome-bot/.android/' 'chrome_infrastructure_adbkey') + # Data should go in android_data_dir, which may be preserved across runs. + android_data_dir = '/sdcard/revenge_of_the_skiabot/' self.device_dirs = default_flavor.DeviceDirs( - dm_dir = self.m.vars.android_data_dir + 'dm_out', - perf_data_dir = self.m.vars.android_data_dir + 'perf', - resource_dir = self.m.vars.android_data_dir + 'resources', - images_dir = self.m.vars.android_data_dir + 'images', - skp_dir = self.m.vars.android_data_dir + 'skps', - svg_dir = self.m.vars.android_data_dir + 'svgs', - tmp_dir = self.m.vars.android_data_dir) + bin_dir = '/data/local/tmp/', + dm_dir = android_data_dir + 'dm_out', + perf_data_dir = android_data_dir + 'perf', + resource_dir = android_data_dir + 'resources', + images_dir = android_data_dir + 'images', + skp_dir = android_data_dir + 'skps', + svg_dir = android_data_dir + 'svgs', + tmp_dir = android_data_dir) # A list of devices we can't root. If rooting fails and a device is not # on the list, we fail the task to avoid perf inconsistencies. @@ -71,7 +74,7 @@ class GNAndroidFlavorUtils(default_flavor.DefaultFlavorUtils): } def _run(self, title, *cmd, **kwargs): - with self.m.context(cwd=self.m.vars.skia_dir): + with self.m.context(cwd=self.m.path['start_dir'].join('skia')): return self.m.run(self.m.step, title, cmd=list(cmd), **kwargs) def _adb(self, title, *cmd, **kwargs): @@ -99,7 +102,7 @@ class GNAndroidFlavorUtils(default_flavor.DefaultFlavorUtils): timeout=180, abort_on_failure=False, fail_build_on_failure=False) - with self.m.context(cwd=self.m.vars.skia_dir): + with self.m.context(cwd=self.m.path['start_dir'].join('skia')): with self.m.env({'ADB_VENDOR_KEYS': self.ADB_PUB_KEY}): return self.m.run.with_retry(self.m.step, title, attempts, cmd=[self.ADB_BINARY]+list(cmd), @@ -427,7 +430,7 @@ wait_for_device() line = line.replace(addr, addr + ' ' + sym.strip()) print line """ % self.ADB_BINARY, - args=[self.m.vars.skia_out.join(self.m.vars.configuration)], + args=[self.m.vars.skia_out], infra_step=True, timeout=300, abort_on_failure=False) @@ -453,17 +456,17 @@ wait_for_device() self._scale_for_nanobench() else: self._scale_for_dm() - app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0]) + app = self.m.vars.skia_out.join(cmd[0]) self._adb('push %s' % cmd[0], - 'push', app, self.m.vars.android_bin_dir) + 'push', app, self.device_dirs.bin_dir) sh = '%s.sh' % cmd[0] self.m.run.writefile(self.m.vars.tmp_dir.join(sh), - 'set -x; %s%s; echo $? >%src' % - (self.m.vars.android_bin_dir, subprocess.list2cmdline(map(str, cmd)), - self.m.vars.android_bin_dir)) + 'set -x; %s%s; echo $? >%src' % ( + self.device_dirs.bin_dir, subprocess.list2cmdline(map(str, cmd)), + self.device_dirs.bin_dir)) self._adb('push %s' % sh, - 'push', self.m.vars.tmp_dir.join(sh), self.m.vars.android_bin_dir) + 'push', self.m.vars.tmp_dir.join(sh), self.device_dirs.bin_dir) self._adb('clear log', 'logcat', '-c') self.m.python.inline('%s' % cmd[0], """ @@ -479,7 +482,7 @@ wait_for_device() print "Couldn't read the return code. Probably killed for OOM." sys.exit(1) """ % (self.ADB_BINARY, self.ADB_BINARY), - args=[self.m.vars.android_bin_dir, sh]) + args=[self.device_dirs.bin_dir, sh]) def copy_file_to_device(self, host, device): self._adb('push %s %s' % (host, device), 'push', host, device) diff --git a/infra/bots/recipe_modules/flavor/gn_chromebook_flavor.py b/infra/bots/recipe_modules/flavor/gn_chromebook_flavor.py index 984eccef25..a133f2bb90 100644 --- a/infra/bots/recipe_modules/flavor/gn_chromebook_flavor.py +++ b/infra/bots/recipe_modules/flavor/gn_chromebook_flavor.py @@ -21,16 +21,16 @@ class GNChromebookFlavorUtils(gn_flavor.GNFlavorUtils): super(GNChromebookFlavorUtils, self).__init__(m) self._user_ip = '' + self.chromeos_homedir = '/home/chronos/user/' self.device_dirs = default_flavor.DeviceDirs( - dm_dir = self.m.vars.chromeos_homedir + 'dm_out', - perf_data_dir = self.m.vars.chromeos_homedir + 'perf', - resource_dir = self.m.vars.chromeos_homedir + 'resources', - images_dir = self.m.vars.chromeos_homedir + 'images', - skp_dir = self.m.vars.chromeos_homedir + 'skps', - svg_dir = self.m.vars.chromeos_homedir + 'svgs', - tmp_dir = self.m.vars.chromeos_homedir) - - self._bin_dir = self.m.vars.chromeos_homedir + 'bin' + bin_dir = self.chromeos_homedir + 'bin', + dm_dir = self.chromeos_homedir + 'dm_out', + perf_data_dir = self.chromeos_homedir + 'perf', + resource_dir = self.chromeos_homedir + 'resources', + images_dir = self.chromeos_homedir + 'images', + skp_dir = self.chromeos_homedir + 'skps', + svg_dir = self.chromeos_homedir + 'svgs', + tmp_dir = self.chromeos_homedir) @property def user_ip(self): @@ -62,10 +62,10 @@ class GNChromebookFlavorUtils(gn_flavor.GNFlavorUtils): self.device_dirs.resource_dir) # Ensure the home dir is marked executable - self._ssh('remount %s as exec' % self.m.vars.chromeos_homedir, + self._ssh('remount %s as exec' % self.chromeos_homedir, 'sudo', 'mount', '-i', '-o', 'remount,exec', '/home/chronos') - self.create_clean_device_dir(self._bin_dir) + self.create_clean_device_dir(self.device_dirs.bin_dir) def create_clean_device_dir(self, path): # use -f to silently return if path doesn't exist @@ -121,13 +121,13 @@ class GNChromebookFlavorUtils(gn_flavor.GNFlavorUtils): name = cmd[0] if name == 'dm': - self.create_clean_host_dir(self.m.vars.dm_dir) + self.create_clean_host_dir(self.host_dirs.dm_dir) if name == 'nanobench': - self.create_clean_host_dir(self.m.vars.perf_data_dir) + self.create_clean_host_dir(self.host_dirs.perf_data_dir) - app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0]) + app = self.m.vars.skia_out.join(cmd[0]) - cmd[0] = '%s/%s' % (self._bin_dir, cmd[0]) + cmd[0] = '%s/%s' % (self.device_dirs.bin_dir, cmd[0]) self.copy_file_to_device(app, cmd[0]) self._ssh('chmod %s' % name, 'chmod', '+x', cmd[0]) diff --git a/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py b/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py index 9ab77eaf35..bd65e01516 100644 --- a/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py +++ b/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py @@ -15,22 +15,21 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils): super(GNChromecastFlavorUtils, self).__init__(m) self._ever_ran_adb = False self._user_ip = '' - self.m.vars.android_bin_dir = self.m.path.join(self.m.vars.android_bin_dir, - 'bin') # Disk space is extremely tight on the Chromecasts (~100M) There is not # enough space on the android_data_dir (/cache/skia) to fit the images, # resources, executable and output the dm images. So, we have dm_out be # on the tempfs (i.e. RAM) /dev/shm. (which is about 140M) - + data_dir = '/cache/skia/' self.device_dirs = default_flavor.DeviceDirs( + bin_dir = '/cache/skia/bin', dm_dir = '/dev/shm/skia/dm_out', - perf_data_dir = self.m.vars.android_data_dir + 'perf', - resource_dir = self.m.vars.android_data_dir + 'resources', - images_dir = self.m.vars.android_data_dir + 'images', - skp_dir = self.m.vars.android_data_dir + 'skps', - svg_dir = self.m.vars.android_data_dir + 'svgs', - tmp_dir = self.m.vars.android_data_dir) + perf_data_dir = data_dir + 'perf', + resource_dir = data_dir + 'resources', + images_dir = data_dir + 'images', + skp_dir = data_dir + 'skps', + svg_dir = data_dir + 'svgs', + tmp_dir = data_dir) @property def user_ip_host(self): @@ -53,8 +52,8 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils): def install(self): super(GNChromecastFlavorUtils, self).install() - self._adb('mkdir ' + self.m.vars.android_bin_dir, - 'shell', 'mkdir', '-p', self.m.vars.android_bin_dir) + self._adb('mkdir ' + self.device_dirs.bin_dir, + 'shell', 'mkdir', '-p', self.device_dirs.bin_dir) def _adb(self, title, *cmd, **kwargs): if not self._ever_ran_adb: @@ -104,7 +103,7 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils): def cleanup_steps(self): if self._ever_ran_adb: # To clean up disk space for next time - self._ssh('Delete executables', 'rm', '-r', self.m.vars.android_bin_dir, + self._ssh('Delete executables', 'rm', '-r', self.device_dirs.bin_dir, abort_on_failure=False, infra_step=True) # Reconnect if was disconnected self._adb('disconnect', 'disconnect') @@ -125,7 +124,7 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils): line = line.replace(addr, addr + ' ' + sym.strip()) print line """, - args=[self.m.vars.skia_out.join(self.m.vars.configuration)], + args=[self.m.vars.skia_out], infra_step=True, abort_on_failure=False) @@ -145,10 +144,10 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils): return self.m.run(self.m.step, title, cmd=ssh_cmd, **kwargs) def step(self, name, cmd, **kwargs): - app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0]) + app = self.m.vars.skia_out.join(cmd[0]) self._adb('push %s' % cmd[0], - 'push', app, self.m.vars.android_bin_dir) + 'push', app, self.device_dirs.bin_dir) - cmd[0] = '%s/%s' % (self.m.vars.android_bin_dir, cmd[0]) + cmd[0] = '%s/%s' % (self.device_dirs.bin_dir, cmd[0]) self._ssh(str(name), *cmd, infra_step=False) diff --git a/infra/bots/recipe_modules/flavor/gn_flavor.py b/infra/bots/recipe_modules/flavor/gn_flavor.py index 8d16015ca4..284f20b365 100644 --- a/infra/bots/recipe_modules/flavor/gn_flavor.py +++ b/infra/bots/recipe_modules/flavor/gn_flavor.py @@ -16,7 +16,7 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): infra_step=infra_step) def step(self, name, cmd): - app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0]) + app = self.device_dirs.bin_dir.join(cmd[0]) cmd = [app] + cmd[1:] env = self.m.context.env path = [] @@ -43,7 +43,8 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): ld_library_path.append(slave_dir.join('linux_vulkan_sdk', 'lib')) if 'SwiftShader' in extra_tokens: - ld_library_path.append(self.m.vars.skia_out.join('swiftshader_out')) + ld_library_path.append( + self.m.vars.build_dir.join('out', 'swiftshader_out')) if 'MSAN' in extra_tokens: # Find the MSAN-built libc++. @@ -60,7 +61,8 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): elif self.m.vars.is_linux: cmd = ['catchsegv'] + cmd elif 'ProcDump' in extra_tokens: - self.m.file.ensure_directory('makedirs dumps', self.m.vars.dumps_dir) + dumps_dir = self.m.path.join(self.m.vars.swarming_out_dir, 'dumps') + self.m.file.ensure_directory('makedirs dumps', dumps_dir) procdump = str(self.m.vars.slave_dir.join('procdump_win', 'procdump64.exe')) # Full docs for ProcDump here: @@ -70,8 +72,7 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): # -e 1 tells procdump to dump once # -x <dump dir> <exe> <args> launches exe and writes dumps to the # specified dir - cmd = [procdump, '-accepteula', '-mp', '-e', '1', - '-x', self.m.vars.dumps_dir] + cmd + cmd = [procdump, '-accepteula', '-mp', '-e', '1', '-x', dumps_dir] + cmd if 'ASAN' in extra_tokens or 'UBSAN' in extra_tokens: if 'Mac' in self.m.vars.builder_cfg.get('os', ''): @@ -104,7 +105,7 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): # Convert path objects or placeholders into strings such that they can # be passed to symbolize_stack_trace.py args = [slave_dir] + [str(x) for x in cmd] - with self.m.context(cwd=self.m.vars.skia_dir, env=env): + with self.m.context(cwd=self.m.path['start_dir'].join('skia'), env=env): self._py('symbolized %s' % name, self.module.resource('symbolize_stack_trace.py'), args=args, diff --git a/infra/bots/recipe_modules/flavor/ios_flavor.py b/infra/bots/recipe_modules/flavor/ios_flavor.py index fb4b8a8adb..3df07451e9 100644 --- a/infra/bots/recipe_modules/flavor/ios_flavor.py +++ b/infra/bots/recipe_modules/flavor/ios_flavor.py @@ -10,6 +10,17 @@ import gn_flavor import os class iOSFlavorUtils(gn_flavor.GNFlavorUtils): + def __init__(self, m): + super(iOSFlavorUtils, self).__init__(m) + self.device_dirs = default_flavor.DeviceDirs( + bin_dir='[unused]', + dm_dir='dm', + perf_data_dir='perf', + resource_dir='resources', + images_dir='images', + skp_dir='skps', + svg_dir='svgs', + tmp_dir='tmp') def install(self): # Set up the device @@ -17,8 +28,7 @@ class iOSFlavorUtils(gn_flavor.GNFlavorUtils): # Install the app. for app_name in ['dm', 'nanobench']: - app_package = self.m.vars.skia_out.join(self.m.vars.configuration, - '%s.app' % app_name) + app_package = self.m.vars.skia_out.join('%s.app' % app_name) def uninstall_app(attempt): # If app ID changes, upgrade will fail, so try uninstalling. @@ -35,15 +45,6 @@ class iOSFlavorUtils(gn_flavor.GNFlavorUtils): between_attempts_fn=uninstall_app, infra_step=True) - self.device_dirs = default_flavor.DeviceDirs( - dm_dir='dm', - perf_data_dir='perf', - resource_dir='resources', - images_dir='images', - skp_dir='skps', - svg_dir='svgs', - tmp_dir='tmp') - def step(self, name, cmd, env=None, **kwargs): bundle_id = 'com.google.%s' % cmd[0] self.m.run(self.m.step, name, @@ -51,7 +52,8 @@ class iOSFlavorUtils(gn_flavor.GNFlavorUtils): map(str, cmd[1:])) def _run_ios_script(self, script, first, *rest): - full = self.m.vars.skia_dir.join('platform_tools/ios/bin/ios_' + script) + full = self.m.path['start_dir'].join( + 'skia', 'platform_tools', 'ios', 'bin', 'ios_' + script) self.m.run(self.m.step, name = '%s %s' % (script, first), cmd = [full, first] + list(rest), @@ -74,7 +76,8 @@ class iOSFlavorUtils(gn_flavor.GNFlavorUtils): self._run_ios_script('mkdir', path) def read_file_on_device(self, path, **kwargs): - full = self.m.vars.skia_dir.join('platform_tools/ios/bin/ios_cat_file') + full = self.m.path['start_dir'].join( + 'skia', 'platform_tools', 'ios', 'bin', 'ios_cat_file') rv = self.m.run(self.m.step, name = 'cat_file %s' % path, cmd = [full, path], diff --git a/infra/bots/recipe_modules/flavor/valgrind_flavor.py b/infra/bots/recipe_modules/flavor/valgrind_flavor.py index 4e3475b19f..b54fcc36dd 100644 --- a/infra/bots/recipe_modules/flavor/valgrind_flavor.py +++ b/infra/bots/recipe_modules/flavor/valgrind_flavor.py @@ -12,8 +12,8 @@ import gn_flavor class ValgrindFlavorUtils(gn_flavor.GNFlavorUtils): def __init__(self, m): super(ValgrindFlavorUtils, self).__init__(m) - self._suppressions_file = self.m.vars.skia_dir.join( - 'tools', 'valgrind.supp') + self._suppressions_file = self.m.path['start_dir'].join( + 'skia', 'tools', 'valgrind.supp') self._valgrind_cipd_dir = self.m.vars.slave_dir.join('valgrind') self._valgrind_fake_dir = self._valgrind_cipd_dir self._valgrind = self._valgrind_fake_dir.join('bin', 'valgrind') @@ -23,7 +23,7 @@ class ValgrindFlavorUtils(gn_flavor.GNFlavorUtils): new_cmd = [self._valgrind, '--gen-suppressions=all', '--leak-check=full', '--track-origins=yes', '--error-exitcode=1', '--num-callers=40', '--suppressions=%s' % self._suppressions_file] - path_to_app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0]) + path_to_app = self.m.vars.skia_out.join(cmd[0]) new_cmd.append(path_to_app) new_cmd.extend(cmd[1:]) with self.m.env({'VALGRIND_LIB': self._lib_dir}): diff --git a/infra/bots/recipe_modules/run/api.py b/infra/bots/recipe_modules/run/api.py index 3464ef7bbf..20dd6c7b85 100644 --- a/infra/bots/recipe_modules/run/api.py +++ b/infra/bots/recipe_modules/run/api.py @@ -51,14 +51,14 @@ class SkiaStepApi(recipe_api.RecipeApi): """Wrapper around api.file.rmtree.""" self.m.file.rmtree('rmtree %s' % self.m.path.basename(path), path) - def asset_version(self, asset_name, test_data=None): + def asset_version(self, asset_name, skia_dir, test_data=None): """Return the contents of VERSION for the given asset as a string. If test_data is not specified, reads the property 'test_<asset_name>_version' or if not present, uses TEST_DEFAULT_ASSET_VERSION.""" - version_file = self.m.vars.infrabots_dir.join( - 'assets', asset_name, 'VERSION') + version_file = skia_dir.join( + 'infra', 'bots', 'assets', asset_name, 'VERSION') if not test_data: test_data = self.m.properties.get( 'test_%s_version' % asset_name, TEST_DEFAULT_ASSET_VERSION) diff --git a/infra/bots/recipe_modules/run/examples/full.py b/infra/bots/recipe_modules/run/examples/full.py index 7f93233514..d6a27c9b96 100644 --- a/infra/bots/recipe_modules/run/examples/full.py +++ b/infra/bots/recipe_modules/run/examples/full.py @@ -35,7 +35,7 @@ def RunSteps(api): api.run.readfile('myfile.txt') api.run.writefile('myfile.txt', 'contents') api.run.rmtree('mydir') - api.run.asset_version('my_asset') + api.run.asset_version('my_asset', api.vars.cache_dir.join('work', 'skia')) # Merge PATHs. with api.context(env={'PATH': 'mydir:%(PATH)s'}): diff --git a/infra/bots/recipe_modules/vars/api.py b/infra/bots/recipe_modules/vars/api.py index e7471089b3..44845a41e8 100644 --- a/infra/bots/recipe_modules/vars/api.py +++ b/infra/bots/recipe_modules/vars/api.py @@ -15,9 +15,6 @@ CONFIG_RELEASE = 'Release' class SkiaVarsApi(recipe_api.RecipeApi): - override_checkout_root = None - override_gclient_cache = None - def setup(self): """Prepare the variables.""" # Setup @@ -27,87 +24,19 @@ class SkiaVarsApi(recipe_api.RecipeApi): # Special input/output directories. self.build_dir = self.slave_dir.join('build') - self.test_dir = self.slave_dir.join('test') - self.perf_dir = self.slave_dir.join('perf') - self.checkout_root = self.slave_dir self.default_env = self.m.context.env self.default_env['CHROME_HEADLESS'] = '1' self.default_env['PATH'] = self.m.path.pathsep.join([ self.default_env.get('PATH', '%(PATH)s'), str(self.m.bot_update._module.PACKAGE_REPO_ROOT), ]) - self.gclient_env = {'DEPOT_TOOLS_UPDATE': '0'} - self.is_compile_bot = self.builder_name.startswith('Build-') - - self.persistent_checkout = False - # Compile bots keep a persistent checkout. - if self.is_compile_bot and 'NoDEPS' not in self.builder_name: - self.persistent_checkout = True - if 'Housekeeper' in self.builder_name: - self.persistent_checkout = True - if '-CT_' in self.builder_name: - self.persistent_checkout = True - # We need the source code for the Coverage's Upload step to be in the - # same absolute location as when we compiled it so we can map the - # coverage data to actual line numbers. We ensure this by making sure - # we have a checkout on the Coverage's Upload step and that the Upload - # step runs on the same bots that Compile. - if 'Coverage' in self.builder_name and 'Upload' in self.builder_name: - self.persistent_checkout = True - self.cache_dir = self.slave_dir.join('cache') - if self.persistent_checkout: - self.checkout_root = self.cache_dir.join('work') - self.gclient_cache = self.cache_dir.join('git') - if self.override_checkout_root: - self.checkout_root = self.override_checkout_root - self.gclient_cache = self.override_gclient_cache - # got_revision is filled in after checkout steps. - self.got_revision = None - else: - # If there's no persistent checkout, then we have to assume we got the - # correct revision of the files from isolate. - self.got_revision = self.m.properties['revision'] - - # Some bots also require a checkout of Flutter; in this case we use the - # checkout of Skia obtained through DEPS in src/third_party/skia. - self.need_flutter_checkout = 'Flutter' in self.builder_name - - self.skia_dir = self.checkout_root.join('skia') - if self.need_flutter_checkout: - self.checkout_root = self.checkout_root.join('flutter') - self.skia_dir = self.checkout_root.join('src', 'third_party', 'skia') - - if not self.persistent_checkout: - self.m.path['checkout'] = self.skia_dir - - self.infrabots_dir = self.skia_dir.join('infra', 'bots') - self.resource_dir = self.skia_dir.join('resources') - self.images_dir = self.slave_dir.join('skimage') - self.skia_out = self.skia_dir.join('out', self.builder_name) + self.swarming_out_dir = self.slave_dir.join( self.m.properties['swarm_out_dir']) - if 'ParentRevision' in self.builder_name: - # Tasks that depend on ParentRevision builds usually also depend on a - # second build task. Use a different path for build results so that the - # binaries end up in different directories in the isolate. - self.swarming_out_dir = self.swarming_out_dir.join('ParentRevision') - self.local_skp_dir = self.slave_dir.join('skp') - self.local_svg_dir = self.slave_dir.join('svg') - if not self.is_compile_bot: - self.skia_out = self.build_dir.join('out') - self.tmp_dir = self.m.path['start_dir'].join('tmp') - # Some bots also require a checkout of chromium. - self.need_chromium_checkout = False - if 'CommandBuffer' in self.builder_name: - self.need_chromium_checkout = True - self.gclient_env['GYP_CHROMIUM_NO_ACTION'] = '0' - if 'RecreateSKPs' in self.builder_name: - self.need_chromium_checkout = True - self.gclient_env['CPPFLAGS'] = ( - '-DSK_ALLOW_CROSSPROCESS_PICTUREIMAGEFILTERS=1') + self.tmp_dir = self.m.path['start_dir'].join('tmp') self.builder_cfg = self.m.builder_name_schema.DictForBuilderName( self.builder_name) @@ -120,6 +49,9 @@ class SkiaVarsApi(recipe_api.RecipeApi): arch = (self.builder_cfg.get('arch') or self.builder_cfg.get('target_arch')) if ('Win' in self.builder_cfg.get('os', '') and arch == 'x86_64'): self.configuration += '_x64' + + self.skia_out = self.build_dir.join('out', self.configuration) + self.extra_tokens = [] if len(self.builder_cfg.get('extra_config', '')) > 0: if self.builder_cfg['extra_config'].startswith('SK'): @@ -138,27 +70,9 @@ class SkiaVarsApi(recipe_api.RecipeApi): self.issue = self.m.properties['patch_issue'] self.patchset = self.m.properties['patch_set'] - self.dm_dir = self.m.path.join( - self.swarming_out_dir, 'dm') - self.perf_data_dir = self.m.path.join(self.swarming_out_dir, - 'perfdata', self.builder_name, 'data') - self.dumps_dir = self.m.path.join( - self.swarming_out_dir, 'dumps') self._swarming_bot_id = None self._swarming_task_id = None - # Data should go under in _data_dir, which may be preserved across runs. - self.android_data_dir = '/sdcard/revenge_of_the_skiabot/' - # Executables go under _bin_dir, which, well, allows executable files. - self.android_bin_dir = '/data/local/tmp/' - - if self.builder_cfg.get('os', '') == 'Chromecast': - # On the Chromecast, everything goes in the (~110M) /cache/skia - self.android_bin_dir = '/cache/skia/' - self.android_data_dir = '/cache/skia/' - - self.chromeos_homedir = '/home/chronos/user/' - # Internal bot support. self.internal_hardware_label = ( self.m.properties.get('internal_hardware_label')) @@ -169,44 +83,6 @@ class SkiaVarsApi(recipe_api.RecipeApi): return 'Ubuntu' in self.builder_name or 'Debian' in self.builder_name @property - def upload_dm_results(self): - # TODO(borenet): Move this into the swarm_test recipe. - skip_upload_bots = [ - 'ASAN', - 'Coverage', - 'MSAN', - 'TSAN', - 'UBSAN', - 'Valgrind', - ] - upload_dm_results = True - for s in skip_upload_bots: - if s in self.m.properties['buildername']: - upload_dm_results = False - break - return upload_dm_results - - @property - def upload_perf_results(self): - # TODO(borenet): Move this into the swarm_perf recipe. - if 'Release' not in self.m.properties['buildername']: - return False - skip_upload_bots = [ - 'ASAN', - 'Coverage', - 'MSAN', - 'TSAN', - 'UBSAN', - 'Valgrind', - ] - upload_perf_results = True - for s in skip_upload_bots: - if s in self.m.properties['buildername']: - upload_perf_results = False - break - return upload_perf_results - - @property def swarming_bot_id(self): if not self._swarming_bot_id: self._swarming_bot_id = self.m.python.inline( diff --git a/infra/bots/recipe_modules/vars/examples/full.expected/Build-Debian9-Clang-x86_64-Release-ParentRevision.json b/infra/bots/recipe_modules/vars/examples/full.expected/Build-Debian9-Clang-x86_64-Release-ParentRevision.json deleted file mode 100644 index 4594f9e6b9..0000000000 --- a/infra/bots/recipe_modules/vars/examples/full.expected/Build-Debian9-Clang-x86_64-Release-ParentRevision.json +++ /dev/null @@ -1,35 +0,0 @@ -[ - { - "cmd": [ - "python", - "-u", - "import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n" - ], - "name": "get swarming bot id", - "stdout": "/path/to/tmp/", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@import os@@@", - "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", - "import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n" - ], - "name": "get swarming task id", - "stdout": "/path/to/tmp/", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@import os@@@", - "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "name": "$result", - "recipe_result": null, - "status_code": 0 - } -]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/vars/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json b/infra/bots/recipe_modules/vars/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json deleted file mode 100644 index 4594f9e6b9..0000000000 --- a/infra/bots/recipe_modules/vars/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json +++ /dev/null @@ -1,35 +0,0 @@ -[ - { - "cmd": [ - "python", - "-u", - "import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n" - ], - "name": "get swarming bot id", - "stdout": "/path/to/tmp/", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@import os@@@", - "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", - "import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n" - ], - "name": "get swarming task id", - "stdout": "/path/to/tmp/", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@import os@@@", - "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "name": "$result", - "recipe_result": null, - "status_code": 0 - } -]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/vars/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json b/infra/bots/recipe_modules/vars/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json deleted file mode 100644 index 4594f9e6b9..0000000000 --- a/infra/bots/recipe_modules/vars/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json +++ /dev/null @@ -1,35 +0,0 @@ -[ - { - "cmd": [ - "python", - "-u", - "import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n" - ], - "name": "get swarming bot id", - "stdout": "/path/to/tmp/", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@import os@@@", - "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", - "import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n" - ], - "name": "get swarming task id", - "stdout": "/path/to/tmp/", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@import os@@@", - "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "name": "$result", - "recipe_result": null, - "status_code": 0 - } -]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/vars/examples/full.py b/infra/bots/recipe_modules/vars/examples/full.py index 29c16c2f71..b32667a339 100644 --- a/infra/bots/recipe_modules/vars/examples/full.py +++ b/infra/bots/recipe_modules/vars/examples/full.py @@ -16,22 +16,17 @@ def RunSteps(api): api.vars.override_gclient_cache = api.path['start_dir'] api.vars.setup() info = [ - api.vars.upload_dm_results, - api.vars.upload_perf_results, api.vars.swarming_bot_id, api.vars.swarming_task_id, ] if api.vars.is_linux: - assert len(info) == 4 # Make pylint happy. + assert len(info) == 2 # Make pylint happy. TEST_BUILDERS = [ 'Build-Debian9-Clang-x86_64-Release-NoDEPS', 'Build-Debian9-Clang-x86_64-Release-Override', - 'Build-Debian9-Clang-x86_64-Release-ParentRevision', 'Build-Debian9-Clang-x86_64-Release-SKNX_NO_SIMD', - 'Build-Debian9-GCC-x86_64-Release-Flutter_Android', - 'Build-Mac-Clang-x86_64-Debug-CommandBuffer', 'Build-Win-Clang-x86_64-Release-Vulkan', 'Housekeeper-Weekly-RecreateSKPs', 'Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Debug-All', |