diff options
Diffstat (limited to 'infra')
46 files changed, 1155 insertions, 672 deletions
diff --git a/infra/bots/recipe_modules/build/android.py b/infra/bots/recipe_modules/build/android.py index 82253bb947..ba57db89a1 100644 --- a/infra/bots/recipe_modules/build/android.py +++ b/infra/bots/recipe_modules/build/android.py @@ -6,7 +6,8 @@ import re -def compile_fn(api, out_dir): +def compile_fn(api, checkout_root, out_dir): + skia_dir = checkout_root.join('skia') compiler = api.vars.builder_cfg.get('compiler') configuration = api.vars.builder_cfg.get('configuration') extra_tokens = api.vars.extra_tokens @@ -34,7 +35,7 @@ def compile_fn(api, out_dir): 'target_cpu': quote(target_arch), } extra_cflags.append('-DDUMMY_ndk_version=%s' % - api.run.asset_version(ndk_asset)) + api.run.asset_version(ndk_asset, skia_dir)) if configuration != 'Debug': args['is_debug'] = 'false' @@ -59,11 +60,11 @@ def compile_fn(api, out_dir): gn_args = ' '.join('%s=%s' % (k,v) for (k,v) in sorted(args.iteritems())) gn = 'gn.exe' if 'Win' in os else 'gn' ninja = 'ninja.exe' if 'Win' in os else 'ninja' - gn = api.vars.skia_dir.join('bin', gn) + gn = skia_dir.join('bin', gn) - with api.context(cwd=api.vars.skia_dir): + with api.context(cwd=skia_dir): api.run(api.python, 'fetch-gn', - script=api.vars.skia_dir.join('bin', 'fetch-gn'), + script=skia_dir.join('bin', 'fetch-gn'), infra_step=True) # If this is the SkQP build, set up the environment and run the script @@ -72,7 +73,7 @@ def compile_fn(api, out_dir): api.infra.update_go_deps() output_binary = out_dir.join('run_testlab') - build_target = api.vars.skia_dir.join('infra', 'cts', 'run_testlab.go') + build_target = skia_dir.join('infra', 'cts', 'run_testlab.go') build_cmd = ['go', 'build', '-o', output_binary, build_target] with api.context(env=api.infra.go_env): api.run(api.step, 'build firebase runner', cmd=build_cmd) @@ -88,8 +89,7 @@ def compile_fn(api, out_dir): 'APK_OUTPUT_DIR': out_dir, } - mk_universal = api.vars.skia_dir.join('tools', 'skqp', - 'make_universal_apk') + mk_universal = skia_dir.join('tools', 'skqp', 'make_universal_apk') with api.context(env=env): api.run(api.step, 'make_universal', cmd=[mk_universal]) else: @@ -99,6 +99,4 @@ def compile_fn(api, out_dir): def copy_extra_build_products(api, src, dst): - if 'SKQP' in api.vars.extra_tokens: - wlist = api.vars.skia_dir.join('infra','cts', 'whitelist_devices.json') - api.file.copy('copy whitelist', wlist, dst) + pass diff --git a/infra/bots/recipe_modules/build/api.py b/infra/bots/recipe_modules/build/api.py index 545f94f288..286190b94a 100644 --- a/infra/bots/recipe_modules/build/api.py +++ b/infra/bots/recipe_modules/build/api.py @@ -36,13 +36,11 @@ class BuildApi(recipe_api.RecipeApi): self.copy_fn = default.copy_extra_build_products super(BuildApi, self).__init__(*args, **kwargs) - def __call__(self): + def __call__(self, checkout_root, out_dir): """Compile the code.""" - out_dir = self.m.vars.skia_out.join(self.m.vars.configuration) - self.compile_fn(self.m, out_dir) + self.compile_fn(self.m, checkout_root, out_dir) - def copy_build_products(self, dst): + def copy_build_products(self, out_dir, dst): """Copy whitelisted build products to dst.""" - src = self.m.vars.skia_out.join(self.m.vars.configuration) - util.copy_whitelisted_build_products(self.m, src, dst) - self.copy_fn(self.m, src, dst) + util.copy_whitelisted_build_products(self.m, out_dir, dst) + self.copy_fn(self.m, out_dir, dst) diff --git a/infra/bots/recipe_modules/build/chromebook.py b/infra/bots/recipe_modules/build/chromebook.py index 0ab624e2bc..f7af5552d3 100644 --- a/infra/bots/recipe_modules/build/chromebook.py +++ b/infra/bots/recipe_modules/build/chromebook.py @@ -3,7 +3,8 @@ # found in the LICENSE file. -def compile_fn(api, out_dir): +def compile_fn(api, checkout_root, out_dir): + skia_dir = checkout_root.join('skia') configuration = api.vars.builder_cfg.get('configuration') os = api.vars.builder_cfg.get('os') target_arch = api.vars.builder_cfg.get('target_arch') @@ -74,7 +75,7 @@ def compile_fn(api, out_dir): 'skia_use_egl': 'true', } extra_cflags.append('-DDUMMY_clang_linux_version=%s' % - api.run.asset_version('clang_linux')) + api.run.asset_version('clang_linux', skia_dir)) if configuration != 'Debug': args['is_debug'] = 'false' @@ -86,11 +87,11 @@ def compile_fn(api, out_dir): gn = 'gn.exe' if 'Win' in os else 'gn' ninja = 'ninja.exe' if 'Win' in os else 'ninja' - gn = api.vars.skia_dir.join('bin', gn) + gn = skia_dir.join('bin', gn) - with api.context(cwd=api.vars.skia_dir, env=env): + with api.context(cwd=skia_dir, env=env): api.run(api.python, 'fetch-gn', - script=api.vars.skia_dir.join('bin', 'fetch-gn'), + script=skia_dir.join('bin', 'fetch-gn'), infra_step=True) api.run(api.step, 'gn gen', cmd=[gn, 'gen', out_dir, '--args=' + gn_args]) api.run(api.step, 'ninja', diff --git a/infra/bots/recipe_modules/build/chromecast.py b/infra/bots/recipe_modules/build/chromecast.py index b1631aa6c0..d0e5b2f4ef 100644 --- a/infra/bots/recipe_modules/build/chromecast.py +++ b/infra/bots/recipe_modules/build/chromecast.py @@ -3,7 +3,8 @@ # found in the LICENSE file. -def compile_fn(api, out_dir): +def compile_fn(api, checkout_root, out_dir): + skia_dir = checkout_root.join('skia') configuration = api.vars.builder_cfg.get('configuration') os = api.vars.builder_cfg.get('os') target_arch = api.vars.builder_cfg.get('target_arch') @@ -21,7 +22,7 @@ def compile_fn(api, out_dir): # Makes the binary small enough to fit on the small disk. '-g0', ('-DDUMMY_cast_toolchain_version=%s' % - api.run.asset_version('cast_toolchain')), + api.run.asset_version('cast_toolchain', skia_dir)), ] extra_ldflags = [ @@ -55,11 +56,11 @@ def compile_fn(api, out_dir): gn = 'gn.exe' if 'Win' in os else 'gn' ninja = 'ninja.exe' if 'Win' in os else 'ninja' - gn = api.vars.skia_dir.join('bin', gn) + gn = skia_dir.join('bin', gn) - with api.context(cwd=api.vars.skia_dir): + with api.context(cwd=skia_dir): api.run(api.python, 'fetch-gn', - script=api.vars.skia_dir.join('bin', 'fetch-gn'), + script=skia_dir.join('bin', 'fetch-gn'), infra_step=True) api.run(api.step, 'gn gen', cmd=[gn, 'gen', out_dir, '--args=' + gn_args]) api.run(api.step, 'ninja', diff --git a/infra/bots/recipe_modules/build/default.py b/infra/bots/recipe_modules/build/default.py index a7ee9801d6..da96dc9e60 100644 --- a/infra/bots/recipe_modules/build/default.py +++ b/infra/bots/recipe_modules/build/default.py @@ -6,12 +6,12 @@ from . import util -def build_command_buffer(api): +def build_command_buffer(api, chrome_dir, skia_dir, out): api.run(api.python, 'build command_buffer', - script=api.vars.skia_dir.join('tools', 'build_command_buffer.py'), + script=skia_dir.join('tools', 'build_command_buffer.py'), args=[ - '--chrome-dir', api.vars.checkout_root, - '--output-dir', api.vars.skia_out.join(api.vars.configuration), + '--chrome-dir', chrome_dir, + '--output-dir', out, '--no-sync', '--no-hooks', '--make-output-dir']) @@ -40,7 +40,8 @@ def compile_swiftshader(api, swiftshader_root, cc, cxx, out): cmd=['ninja', '-C', out, 'libEGL.so', 'libGLESv2.so']) -def compile_fn(api, out_dir): +def compile_fn(api, checkout_root, out_dir): + skia_dir = checkout_root.join('skia') compiler = api.vars.builder_cfg.get('compiler', '') configuration = api.vars.builder_cfg.get('configuration', '') extra_tokens = api.vars.extra_tokens @@ -69,7 +70,7 @@ def compile_fn(api, out_dir): extra_ldflags.append('-B%s/bin' % clang_linux) extra_ldflags.append('-fuse-ld=lld') extra_cflags.append('-DDUMMY_clang_linux_version=%s' % - api.run.asset_version('clang_linux')) + api.run.asset_version('clang_linux', skia_dir)) if os == 'Ubuntu14': extra_ldflags.extend(['-static-libstdc++', '-static-libgcc']) @@ -89,7 +90,7 @@ def compile_fn(api, out_dir): '-Wno-format-truncation', '-Wno-uninitialized', ('-DDUMMY_mips64el_toolchain_linux_version=%s' % - api.run.asset_version('mips64el_toolchain_linux')) + api.run.asset_version('mips64el_toolchain_linux', skia_dir)) ]) if configuration == 'Release': # This warning is only triggered when fuzz_canvas is inlined. @@ -106,7 +107,7 @@ def compile_fn(api, out_dir): cxx = emscripten_sdk + '/emscripten/incoming/em++' extra_cflags.append('-Wno-unknown-warning-option') extra_cflags.append('-DDUMMY_emscripten_sdk_version=%s' % - api.run.asset_version('emscripten_sdk')) + api.run.asset_version('emscripten_sdk', skia_dir)) if 'Coverage' in extra_tokens: # See https://clang.llvm.org/docs/SourceBasedCodeCoverage.html for # more info on using llvm to gather coverage information. @@ -139,8 +140,7 @@ def compile_fn(api, out_dir): if 'ANGLE' in extra_tokens: args['skia_use_angle'] = 'true' if 'SwiftShader' in extra_tokens: - swiftshader_root = api.vars.skia_dir.join('third_party', 'externals', - 'swiftshader') + swiftshader_root = skia_dir.join('third_party', 'externals', 'swiftshader') swiftshader_out = out_dir.join('swiftshader_out') compile_swiftshader(api, swiftshader_root, cc, cxx, swiftshader_out) args['skia_use_egl'] = 'true' @@ -154,7 +154,8 @@ def compile_fn(api, out_dir): '-L%s' % swiftshader_out, ]) if 'CommandBuffer' in extra_tokens: - api.run.run_once(build_command_buffer, api) + chrome_dir = checkout_root + api.run.run_once(build_command_buffer, api, chrome_dir, skia_dir, out_dir) if 'MSAN' in extra_tokens: args['skia_enable_gpu'] = 'false' args['skia_use_fontconfig'] = 'false' @@ -215,7 +216,7 @@ def compile_fn(api, out_dir): if compiler == 'Clang' and 'Win' in os: args['clang_win'] = '"%s"' % api.vars.slave_dir.join('clang_win') extra_cflags.append('-DDUMMY_clang_win_version=%s' % - api.run.asset_version('clang_win')) + api.run.asset_version('clang_win', skia_dir)) if target_arch == 'wasm': args.update({ 'skia_use_freetype': 'false', @@ -253,18 +254,18 @@ def compile_fn(api, out_dir): gn = 'gn.exe' if 'Win' in os else 'gn' ninja = 'ninja.exe' if 'Win' in os else 'ninja' - gn = api.vars.skia_dir.join('bin', gn) + gn = skia_dir.join('bin', gn) - with api.context(cwd=api.vars.skia_dir): + with api.context(cwd=skia_dir): api.run(api.python, 'fetch-gn', - script=api.vars.skia_dir.join('bin', 'fetch-gn'), + script=skia_dir.join('bin', 'fetch-gn'), infra_step=True) if 'CheckGeneratedFiles' in extra_tokens: - env['PATH'] = '%s:%%(PATH)s' % api.vars.skia_dir.join('bin') + env['PATH'] = '%s:%%(PATH)s' % skia_dir.join('bin') api.run(api.python, 'fetch-clang-format', - script=api.vars.skia_dir.join('bin', 'fetch-clang-format'), + script=skia_dir.join('bin', 'fetch-clang-format'), infra_step=True) if target_arch == 'wasm': fastcomp = emscripten_sdk + '/clang/fastcomp/build_incoming_64/bin' diff --git a/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-Clang-universal-devrel-Android_SKQP.json b/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-Clang-universal-devrel-Android_SKQP.json index 762dc757d5..f98a77e515 100644 --- a/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-Clang-universal-devrel-Android_SKQP.json +++ b/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-Clang-universal-devrel-Android_SKQP.json @@ -142,20 +142,6 @@ ] }, { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "[START_DIR]/cache/work/skia/infra/cts/whitelist_devices.json", - "[START_DIR]/[SWARM_OUT_DIR]/out/devrel" - ], - "infra_step": true, - "name": "copy whitelist" - }, - { "name": "$result", "recipe_result": null, "status_code": 0 diff --git a/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json b/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json index 3be7441907..90d0df0f2d 100644 --- a/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json +++ b/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json @@ -30,7 +30,7 @@ "cmd": [ "ninja", "-C", - "out/android_release", + "[START_DIR]/cache/work/flutter/src/out/android_release", "-j100" ], "cwd": "[START_DIR]/cache/work/flutter/src", @@ -45,7 +45,7 @@ "python", "-u", "import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n", - "[START_DIR]/cache/work/flutter/src/third_party/skia/out/Build-Debian9-GCC-x86_64-Release-Flutter_Android/Release", + "[START_DIR]/cache/work/flutter/src/out/android_release", "[START_DIR]/[SWARM_OUT_DIR]/out/Release" ], "infra_step": true, diff --git a/infra/bots/recipe_modules/build/examples/full.py b/infra/bots/recipe_modules/build/examples/full.py index 8b801410f5..aca20a6195 100644 --- a/infra/bots/recipe_modules/build/examples/full.py +++ b/infra/bots/recipe_modules/build/examples/full.py @@ -5,6 +5,7 @@ DEPS = [ 'build', + 'recipe_engine/path', 'recipe_engine/properties', 'recipe_engine/raw_io', 'run', @@ -15,10 +16,24 @@ DEPS = [ def RunSteps(api): api.vars.setup() - api.build() - api.build.copy_build_products( - api.vars.swarming_out_dir.join( - 'out', api.vars.configuration)) + # Hackery to avoid changing expectations. + checkout_root = api.vars.cache_dir.join('work') + out_dir = checkout_root.join( + 'skia', 'out', api.vars.builder_name, api.vars.configuration) + if ('CheckGeneratedFiles' in api.vars.builder_name or + '-CT_' in api.vars.builder_name): + out_dir = api.vars.build_dir.join('out', api.vars.configuration) + elif 'NoDEPS' in api.vars.builder_name: + checkout_root = api.path['start_dir'] + out_dir = checkout_root.join( + 'skia', 'out', api.vars.builder_name, api.vars.configuration) + elif 'Flutter' in api.vars.builder_name: + checkout_root = checkout_root.join('flutter') + out_dir = checkout_root.join('src', 'out', 'android_release') + + api.build(checkout_root=checkout_root, out_dir=out_dir) + dst = api.vars.swarming_out_dir.join('out', api.vars.configuration) + api.build.copy_build_products(out_dir=out_dir, dst=dst) api.run.check_failure() diff --git a/infra/bots/recipe_modules/build/flutter.py b/infra/bots/recipe_modules/build/flutter.py index fee92c1621..e1fc52e25f 100644 --- a/infra/bots/recipe_modules/build/flutter.py +++ b/infra/bots/recipe_modules/build/flutter.py @@ -3,11 +3,10 @@ # found in the LICENSE file. -def compile_fn(api, _): - flutter_dir = api.vars.checkout_root.join('src') +def compile_fn(api, checkout_root, out_dir): + flutter_dir = checkout_root.join('src') configuration = api.vars.builder_cfg.get('configuration').lower() extra_tokens = api.vars.extra_tokens - out_dir = configuration with api.context(cwd=flutter_dir): # Setup GN args. @@ -16,10 +15,9 @@ def compile_fn(api, _): ] if 'Android' in extra_tokens: gn_args.append('--android') - out_dir = 'android_' + out_dir # Delete out_dir so that we start from a clean slate. See skbug/6310. - api.run.rmtree(flutter_dir.join('out', out_dir)) + api.run.rmtree(out_dir) # Run GN. api.run( @@ -31,7 +29,7 @@ def compile_fn(api, _): api.run( api.step, 'build_flutter', - cmd=['ninja', '-C', 'out/' + out_dir, '-j100']) + cmd=['ninja', '-C', out_dir, '-j100']) def copy_extra_build_products(api, src, dst): diff --git a/infra/bots/recipe_modules/core/api.py b/infra/bots/recipe_modules/core/api.py index f7b5113255..f943b865b2 100644 --- a/infra/bots/recipe_modules/core/api.py +++ b/infra/bots/recipe_modules/core/api.py @@ -17,39 +17,58 @@ from recipe_engine import config_types class SkiaApi(recipe_api.RecipeApi): + @property + def default_checkout_root(self): + """The default location for cached persistent checkouts.""" + return self.m.vars.cache_dir.join('work') + def patch_ref(self, issue, patchset): """Build a ref for the given issue and patchset.""" return 'refs/changes/%s/%s/%s' % (issue[-2:], issue, patchset) - def checkout_git(self): + def checkout_git(self, checkout_root): """Run the steps to perform a pure-git checkout without DEPS.""" + skia_dir = checkout_root.join('skia') self.m.git.checkout( - self.m.properties['repository'], dir_path=self.m.vars.skia_dir, + self.m.properties['repository'], dir_path=skia_dir, ref=self.m.properties['revision'], submodules=False) if self.m.vars.is_trybot: ref = self.patch_ref(str(self.m.vars.issue), str(self.m.vars.patchset)) self.m.git('fetch', 'origin', ref) self.m.git('checkout', 'FETCH_HEAD') self.m.git('rebase', self.m.properties['revision']) + return self.m.properties['revision'] - def checkout_bot_update(self): + def checkout_bot_update(self, checkout_root, gclient_cache=None): """Run the steps to obtain a checkout using bot_update.""" + if not gclient_cache: + gclient_cache = self.m.vars.cache_dir.join('git') + cfg_kwargs = {} is_parent_revision = 'ParentRevision' in self.m.vars.extra_tokens # Use a persistent gclient cache for Swarming. - cfg_kwargs['CACHE_DIR'] = self.m.vars.gclient_cache + cfg_kwargs['CACHE_DIR'] = gclient_cache # Create the checkout path if necessary. - if not self.m.path.exists(self.m.vars.checkout_root): - self.m.file.ensure_directory('makedirs checkout_path', - self.m.vars.checkout_root) + if not self.m.path.exists(checkout_root): + self.m.file.ensure_directory('makedirs checkout_path', checkout_root) # Initial cleanup. gclient_cfg = self.m.gclient.make_config(**cfg_kwargs) + # Some bots also require a checkout of chromium. + need_chromium_checkout = False + gclient_env = {'DEPOT_TOOLS_UPDATE': '0'} + if 'CommandBuffer' in self.m.properties['buildername']: + need_chromium_checkout = True + gclient_env['GYP_CHROMIUM_NO_ACTION'] = '0' + if 'RecreateSKPs' in self.m.properties['buildername']: + need_chromium_checkout = True + gclient_env['CPPFLAGS'] = '-DSK_ALLOW_CROSSPROCESS_PICTUREIMAGEFILTERS=1' + # Add chromium first because of skbug.com/7917. - if self.m.vars.need_chromium_checkout: + if need_chromium_checkout: chromium = gclient_cfg.solutions.add() chromium.name = 'src' chromium.managed = False @@ -57,14 +76,16 @@ class SkiaApi(recipe_api.RecipeApi): chromium.revision = 'origin/master' main_repo = self.m.properties['repository'] - if self.m.vars.need_flutter_checkout: + + need_flutter_checkout = 'Flutter' in self.m.properties['buildername'] + if need_flutter_checkout: main_repo = 'https://github.com/flutter/engine.git' main_name = self.m.path.basename(main_repo) if main_name.endswith('.git'): main_name = main_name[:-len('.git')] # Special case for flutter because it seems to need a very specific # directory structure to successfully build. - if self.m.vars.need_flutter_checkout and main_name == 'engine': + if need_flutter_checkout and main_name == 'engine': main_name = 'src/flutter' main = gclient_cfg.solutions.add() main.name = main_name @@ -81,7 +102,7 @@ class SkiaApi(recipe_api.RecipeApi): if patch_root.endswith('.git'): patch_root = patch_root[:-4] - if self.m.vars.need_flutter_checkout: + if need_flutter_checkout: # Skia is a DEP of Flutter; the 'revision' property is a Skia revision, # and any patch should be applied to Skia, not Flutter. main.revision = 'origin/master' @@ -99,7 +120,7 @@ class SkiaApi(recipe_api.RecipeApi): # TODO(rmistry): Remove the below block after there is a solution for # crbug.com/616443 - entries_file = self.m.vars.checkout_root.join('.gclient_entries') + entries_file = checkout_root.join('.gclient_entries') if self.m.path.exists(entries_file) or self._test_data.enabled: self.m.file.remove('remove %s' % entries_file, entries_file) @@ -118,7 +139,7 @@ class SkiaApi(recipe_api.RecipeApi): main.revision = main.revision + '^' self.m.gclient.c = gclient_cfg - with self.m.context(cwd=self.m.vars.checkout_root): + with self.m.context(cwd=checkout_root): update_step = self.m.bot_update.ensure_checkout( patch_root=patch_root, # The logic in ensure_checkout for this arg is fairly naive, so if @@ -128,10 +149,7 @@ class SkiaApi(recipe_api.RecipeApi): patch=not (self.m.vars.is_trybot and is_parent_revision) ) - self.m.vars.got_revision = ( - update_step.presentation.properties['got_revision']) - - if self.m.vars.need_chromium_checkout or self.m.vars.need_flutter_checkout: - with self.m.context(cwd=self.m.vars.checkout_root, - env=self.m.vars.gclient_env): + if need_chromium_checkout or need_flutter_checkout: + with self.m.context(cwd=checkout_root, env=gclient_env): self.m.gclient.runhooks() + return update_step.presentation.properties['got_revision'] diff --git a/infra/bots/recipe_modules/core/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json b/infra/bots/recipe_modules/core/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json new file mode 100644 index 0000000000..a066ac5a97 --- /dev/null +++ b/infra/bots/recipe_modules/core/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json @@ -0,0 +1,147 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/cache/work/flutter" + ], + "infra_step": true, + "name": "makedirs checkout_path" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "remove", + "[START_DIR]/cache/work/flutter/.gclient_entries" + ], + "infra_step": true, + "name": "remove [START_DIR]/cache/work/flutter/.gclient_entries" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec-path", + "cache_dir = '[START_DIR]/cache/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': True, 'name': 'src/flutter', 'url': 'https://github.com/flutter/engine.git'}]\ntarget_os = ['android']", + "--patch_root", + "src/third_party/skia", + "--revision_mapping_file", + "{\"got_flutter_revision\": \"src/flutter\", \"got_revision\": \"src/third_party/skia\"}", + "--git-cache-dir", + "[START_DIR]/cache/git", + "--cleanup-dir", + "[CLEANUP]/bot_update", + "--output_json", + "/path/to/tmp/json", + "--revision", + "src/flutter@origin/master", + "--revision", + "src/third_party/skia@abc123" + ], + "cwd": "[START_DIR]/cache/work/flutter", + "env_prefixes": { + "PATH": [ + "RECIPE_PACKAGE_REPO[depot_tools]" + ] + }, + "infra_step": true, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src/flutter\": \"origin/master\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"src/third_party/skia\": \"abc123\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src/flutter\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/src/flutter.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9221bca00ddbd888260084def81f09543281b952\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"src/third_party/skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/src/third_party/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"d9c4a4d173a97ef2832b65636b4200bb93ea8ee1\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src/third_party/skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_flutter_revision\": \"9221bca00ddbd888260084def81f09543281b952\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_flutter_revision_cp\": \"refs/heads/master@{#84512}\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"d9c4a4d173a97ef2832b65636b4200bb93ea8ee1\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#143121}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"src/flutter\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"src/flutter\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/src/flutter.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9221bca00ddbd888260084def81f09543281b952\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"src/third_party/skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/src/third_party/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"d9c4a4d173a97ef2832b65636b4200bb93ea8ee1\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_flutter_revision@\"9221bca00ddbd888260084def81f09543281b952\"@@@", + "@@@SET_BUILD_PROPERTY@got_flutter_revision_cp@\"refs/heads/master@{#84512}\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"d9c4a4d173a97ef2832b65636b4200bb93ea8ee1\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#143121}\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py", + "runhooks" + ], + "cwd": "[START_DIR]/cache/work/flutter", + "env": { + "DEPOT_TOOLS_UPDATE": "0", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "gclient runhooks" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/tmp" + ], + "infra_step": true, + "name": "makedirs tmp_dir" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/core/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json b/infra/bots/recipe_modules/core/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json new file mode 100644 index 0000000000..d9db2bbf14 --- /dev/null +++ b/infra/bots/recipe_modules/core/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json @@ -0,0 +1,134 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/cache/work" + ], + "infra_step": true, + "name": "makedirs checkout_path" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "remove", + "[START_DIR]/cache/work/.gclient_entries" + ], + "infra_step": true, + "name": "remove [START_DIR]/cache/work/.gclient_entries" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec-path", + "cache_dir = '[START_DIR]/cache/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'src', 'url': 'https://chromium.googlesource.com/chromium/src.git'}, {'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]", + "--patch_root", + "skia", + "--revision_mapping_file", + "{\"got_revision\": \"skia\"}", + "--git-cache-dir", + "[START_DIR]/cache/git", + "--cleanup-dir", + "[CLEANUP]/bot_update", + "--output_json", + "/path/to/tmp/json", + "--revision", + "skia@abc123", + "--revision", + "src@origin/master" + ], + "cwd": "[START_DIR]/cache/work", + "env_prefixes": { + "PATH": [ + "RECIPE_PACKAGE_REPO[depot_tools]" + ] + }, + "infra_step": true, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"src\": \"origin/master\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"src\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py", + "runhooks" + ], + "cwd": "[START_DIR]/cache/work", + "env": { + "DEPOT_TOOLS_UPDATE": "0", + "GYP_CHROMIUM_NO_ACTION": "0", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "gclient runhooks" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/tmp" + ], + "infra_step": true, + "name": "makedirs tmp_dir" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/core/examples/full.py b/infra/bots/recipe_modules/core/examples/full.py index f3cfb29ca7..ef12119015 100644 --- a/infra/bots/recipe_modules/core/examples/full.py +++ b/infra/bots/recipe_modules/core/examples/full.py @@ -15,13 +15,18 @@ DEPS = [ def RunSteps(api): api.vars.setup() + bot_update = True if 'NoDEPS' in api.properties['buildername']: bot_update = False + if bot_update: - api.core.checkout_bot_update() + checkout_root = api.core.default_checkout_root + if 'Flutter' in api.vars.builder_name: + checkout_root = checkout_root.join('flutter') + api.core.checkout_bot_update(checkout_root=checkout_root) else: - api.core.checkout_git() + api.core.checkout_git(checkout_root=api.path['start_dir']) api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) @@ -139,3 +144,23 @@ def GenTests(api): api.path['start_dir'].join('tmp', 'uninteresting_hashes.txt') ) ) + + buildername = 'Build-Mac-Clang-x86_64-Debug-CommandBuffer' + yield ( + api.test(buildername) + + api.properties(buildername=buildername, + repository='https://skia.googlesource.com/skia.git', + revision='abc123', + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + ) + + buildername = 'Build-Debian9-GCC-x86_64-Release-Flutter_Android' + yield ( + api.test(buildername) + + api.properties(buildername=buildername, + repository='https://skia.googlesource.com/skia.git', + revision='abc123', + path_config='kitchen', + swarm_out_dir='[SWARM_OUT_DIR]') + ) diff --git a/infra/bots/recipe_modules/flavor/api.py b/infra/bots/recipe_modules/flavor/api.py index f432066e9a..c5a1be8030 100644 --- a/infra/bots/recipe_modules/flavor/api.py +++ b/infra/bots/recipe_modules/flavor/api.py @@ -64,6 +64,9 @@ class SkiaFlavorApi(recipe_api.RecipeApi): def setup(self): self._f = self.get_flavor(self.m.vars) + self.device_dirs = self._f.device_dirs + self.host_dirs = self._f.host_dirs + self._skia_dir = self.m.path['start_dir'].join('skia') def step(self, name, cmd, **kwargs): return self._f.step(name, cmd, **kwargs) @@ -97,12 +100,11 @@ class SkiaFlavorApi(recipe_api.RecipeApi): def install(self, skps=False, images=False, svgs=False, resources=False): self._f.install() - self.device_dirs = self._f.device_dirs # TODO(borenet): Only copy files which have changed. if resources: self.copy_directory_contents_to_device( - self.m.vars.resource_dir, + self.m.path['start_dir'].join('skia', 'resources'), self.device_dirs.resource_dir) if skps: @@ -138,7 +140,7 @@ class SkiaFlavorApi(recipe_api.RecipeApi): def _copy_images(self): """Download and copy test images if needed.""" - version = self.m.run.asset_version('skimage') + version = self.m.run.asset_version('skimage', self._skia_dir) self.m.run.writefile( self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_SK_IMAGE), version) @@ -146,13 +148,13 @@ class SkiaFlavorApi(recipe_api.RecipeApi): version, VERSION_FILE_SK_IMAGE, self.m.vars.tmp_dir, - self.m.vars.images_dir, + self.host_dirs.images_dir, self.device_dirs.images_dir) return version def _copy_skps(self): """Download and copy the SKPs if needed.""" - version = self.m.run.asset_version('skp') + version = self.m.run.asset_version('skp', self._skia_dir) self.m.run.writefile( self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_SKP), version) @@ -160,13 +162,13 @@ class SkiaFlavorApi(recipe_api.RecipeApi): version, VERSION_FILE_SKP, self.m.vars.tmp_dir, - self.m.vars.local_skp_dir, + self.host_dirs.skp_dir, self.device_dirs.skp_dir) return version def _copy_svgs(self): """Download and copy the SVGs if needed.""" - version = self.m.run.asset_version('svg') + version = self.m.run.asset_version('svg', self._skia_dir) self.m.run.writefile( self.m.path.join(self.m.vars.tmp_dir, VERSION_FILE_SVG), version) @@ -174,6 +176,6 @@ class SkiaFlavorApi(recipe_api.RecipeApi): version, VERSION_FILE_SVG, self.m.vars.tmp_dir, - self.m.vars.local_svg_dir, + self.host_dirs.svg_dir, self.device_dirs.svg_dir) return version diff --git a/infra/bots/recipe_modules/flavor/default_flavor.py b/infra/bots/recipe_modules/flavor/default_flavor.py index 7ce61775ac..ec684efd1f 100644 --- a/infra/bots/recipe_modules/flavor/default_flavor.py +++ b/infra/bots/recipe_modules/flavor/default_flavor.py @@ -17,6 +17,7 @@ WIN_TOOLCHAIN_DIR = 't' class DeviceDirs(object): def __init__(self, + bin_dir, dm_dir, perf_data_dir, resource_dir, @@ -24,6 +25,7 @@ class DeviceDirs(object): skp_dir, svg_dir, tmp_dir): + self._bin_dir = bin_dir self._dm_dir = dm_dir self._perf_data_dir = perf_data_dir self._resource_dir = resource_dir @@ -33,6 +35,10 @@ class DeviceDirs(object): self._tmp_dir = tmp_dir @property + def bin_dir(self): + return self._bin_dir + + @property def dm_dir(self): """Where DM writes.""" return self._dm_dir @@ -85,11 +91,18 @@ class DefaultFlavorUtils(object): # syntax as regular recipe modules to run steps, eg: self.m.step(...) self.m = module.m self._chrome_path = None - self._win_toolchain_dir = self.m.vars.slave_dir.join(WIN_TOOLCHAIN_DIR) - win_toolchain_asset_path = self.m.vars.infrabots_dir.join( - 'assets', 'win_toolchain', 'VERSION') - if not self.m.path.exists(win_toolchain_asset_path): - self._win_toolchain_dir = self.m.vars.slave_dir + self.device_dirs = DeviceDirs( + bin_dir=self.m.vars.build_dir.join('out', self.m.vars.configuration), + dm_dir=self.m.path.join(self.m.vars.swarming_out_dir, 'dm'), + perf_data_dir=self.m.path.join( + self.m.vars.swarming_out_dir, + 'perfdata', self.m.vars.builder_name, 'data'), + resource_dir=self.m.path['start_dir'].join('skia', 'resources'), + images_dir=self.m.path['start_dir'].join('skimage'), + skp_dir=self.m.path['start_dir'].join('skp'), + svg_dir=self.m.path['start_dir'].join('svg'), + tmp_dir=self.m.vars.tmp_dir) + self.host_dirs = self.device_dirs def device_path_join(self, *args): """Like os.path.join(), but for paths on a connected device.""" @@ -137,14 +150,7 @@ class DefaultFlavorUtils(object): def install(self): """Run device-specific installation steps.""" - self.device_dirs = DeviceDirs( - dm_dir=self.m.vars.dm_dir, - perf_data_dir=self.m.vars.perf_data_dir, - resource_dir=self.m.vars.resource_dir, - images_dir=self.m.vars.images_dir, - skp_dir=self.m.vars.local_skp_dir, - svg_dir=self.m.vars.local_svg_dir, - tmp_dir=self.m.vars.tmp_dir) + pass def cleanup_steps(self): """Run any device-specific cleanup steps.""" diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/Perf-Ubuntu14-Clang-GCE-CPU-AVX2-x86_64-Release-All-CT_BENCH_1k_SKPs.json b/infra/bots/recipe_modules/flavor/examples/full.expected/Perf-Ubuntu14-Clang-GCE-CPU-AVX2-x86_64-Release-All-CT_BENCH_1k_SKPs.json deleted file mode 100644 index bd783229ba..0000000000 --- a/infra/bots/recipe_modules/flavor/examples/full.expected/Perf-Ubuntu14-Clang-GCE-CPU-AVX2-x86_64-Release-All-CT_BENCH_1k_SKPs.json +++ /dev/null @@ -1,164 +0,0 @@ -[ - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "rmtree", - "results_dir" - ], - "infra_step": true, - "name": "rmtree results_dir" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "ensure-directory", - "--mode", - "0777", - "results_dir" - ], - "infra_step": true, - "name": "makedirs results_dir" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "rmtree", - "device_results_dir" - ], - "infra_step": true, - "name": "rmtree device_results_dir" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "ensure-directory", - "--mode", - "0777", - "device_results_dir" - ], - "infra_step": true, - "name": "makedirs device_results_dir" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "[START_DIR]/cache/work/skia/infra/bots/assets/skp/VERSION", - "/path/to/tmp/" - ], - "infra_step": true, - "name": "Get skp VERSION" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "42", - "[START_DIR]/tmp/SKP_VERSION" - ], - "infra_step": true, - "name": "write SKP_VERSION" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "[START_DIR]/cache/work/skia/infra/bots/assets/skimage/VERSION", - "/path/to/tmp/" - ], - "infra_step": true, - "name": "Get skimage VERSION" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "42", - "[START_DIR]/tmp/SK_IMAGE_VERSION" - ], - "infra_step": true, - "name": "write SK_IMAGE_VERSION" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "[START_DIR]/cache/work/skia/infra/bots/assets/svg/VERSION", - "/path/to/tmp/" - ], - "infra_step": true, - "name": "Get svg VERSION" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "42", - "[START_DIR]/tmp/SVG_VERSION" - ], - "infra_step": true, - "name": "write SVG_VERSION" - }, - { - "cmd": [ - "python", - "-u", - "RECIPE_MODULE[skia::flavor]/resources/symbolize_stack_trace.py", - "[START_DIR]", - "catchsegv", - "[START_DIR]/build/out/Release/nanobench", - "--some-flag" - ], - "cwd": "[START_DIR]/cache/work/skia", - "env": { - "CHROME_HEADLESS": "1", - "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" - }, - "name": "symbolized nanobench" - }, - { - "name": "$result", - "recipe_result": null, - "status_code": 0 - } -]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/flavor/examples/full.py b/infra/bots/recipe_modules/flavor/examples/full.py index 5d30a4f29b..bce549aeb3 100644 --- a/infra/bots/recipe_modules/flavor/examples/full.py +++ b/infra/bots/recipe_modules/flavor/examples/full.py @@ -43,11 +43,12 @@ def RunSteps(api): if 'Test' in api.properties['buildername']: api.flavor.step('dm', ['dm', '--some-flag']) api.flavor.copy_directory_contents_to_host( - api.flavor.device_dirs.dm_dir, api.vars.dm_dir) + api.flavor.device_dirs.dm_dir, api.flavor.host_dirs.dm_dir) elif 'Perf' in api.properties['buildername']: api.flavor.step('nanobench', ['nanobench', '--some-flag']) api.flavor.copy_directory_contents_to_host( - api.flavor.device_dirs.perf_data_dir, api.vars.perf_data_dir) + api.flavor.device_dirs.perf_data_dir, + api.flavor.host_dirs.perf_data_dir) finally: api.flavor.cleanup_steps() api.run.check_failure() @@ -62,7 +63,6 @@ TEST_BUILDERS = [ 'Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Release-All', 'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Debug-All-MSAN', 'Perf-Debian9-Clang-GCE-CPU-AVX2-x86_64-Release-All-ASAN', - 'Perf-Ubuntu14-Clang-GCE-CPU-AVX2-x86_64-Release-All-CT_BENCH_1k_SKPs', 'Test-Android-Clang-AndroidOne-GPU-Mali400MP2-arm-Release-All-Android', 'Test-Android-Clang-GalaxyS7_G930FD-GPU-MaliT880-arm64-Debug-All-Android', 'Test-Android-Clang-Nexus5x-GPU-Adreno418-arm64-Debug-All-Android', diff --git a/infra/bots/recipe_modules/flavor/gn_android_flavor.py b/infra/bots/recipe_modules/flavor/gn_android_flavor.py index 3f83c6d226..8348053407 100644 --- a/infra/bots/recipe_modules/flavor/gn_android_flavor.py +++ b/infra/bots/recipe_modules/flavor/gn_android_flavor.py @@ -22,14 +22,17 @@ class GNAndroidFlavorUtils(default_flavor.DefaultFlavorUtils): self.ADB_PUB_KEY = ('/home/chrome-bot/.android/' 'chrome_infrastructure_adbkey') + # Data should go in android_data_dir, which may be preserved across runs. + android_data_dir = '/sdcard/revenge_of_the_skiabot/' self.device_dirs = default_flavor.DeviceDirs( - dm_dir = self.m.vars.android_data_dir + 'dm_out', - perf_data_dir = self.m.vars.android_data_dir + 'perf', - resource_dir = self.m.vars.android_data_dir + 'resources', - images_dir = self.m.vars.android_data_dir + 'images', - skp_dir = self.m.vars.android_data_dir + 'skps', - svg_dir = self.m.vars.android_data_dir + 'svgs', - tmp_dir = self.m.vars.android_data_dir) + bin_dir = '/data/local/tmp/', + dm_dir = android_data_dir + 'dm_out', + perf_data_dir = android_data_dir + 'perf', + resource_dir = android_data_dir + 'resources', + images_dir = android_data_dir + 'images', + skp_dir = android_data_dir + 'skps', + svg_dir = android_data_dir + 'svgs', + tmp_dir = android_data_dir) # A list of devices we can't root. If rooting fails and a device is not # on the list, we fail the task to avoid perf inconsistencies. @@ -71,7 +74,7 @@ class GNAndroidFlavorUtils(default_flavor.DefaultFlavorUtils): } def _run(self, title, *cmd, **kwargs): - with self.m.context(cwd=self.m.vars.skia_dir): + with self.m.context(cwd=self.m.path['start_dir'].join('skia')): return self.m.run(self.m.step, title, cmd=list(cmd), **kwargs) def _adb(self, title, *cmd, **kwargs): @@ -99,7 +102,7 @@ class GNAndroidFlavorUtils(default_flavor.DefaultFlavorUtils): timeout=180, abort_on_failure=False, fail_build_on_failure=False) - with self.m.context(cwd=self.m.vars.skia_dir): + with self.m.context(cwd=self.m.path['start_dir'].join('skia')): with self.m.env({'ADB_VENDOR_KEYS': self.ADB_PUB_KEY}): return self.m.run.with_retry(self.m.step, title, attempts, cmd=[self.ADB_BINARY]+list(cmd), @@ -427,7 +430,7 @@ wait_for_device() line = line.replace(addr, addr + ' ' + sym.strip()) print line """ % self.ADB_BINARY, - args=[self.m.vars.skia_out.join(self.m.vars.configuration)], + args=[self.m.vars.skia_out], infra_step=True, timeout=300, abort_on_failure=False) @@ -453,17 +456,17 @@ wait_for_device() self._scale_for_nanobench() else: self._scale_for_dm() - app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0]) + app = self.m.vars.skia_out.join(cmd[0]) self._adb('push %s' % cmd[0], - 'push', app, self.m.vars.android_bin_dir) + 'push', app, self.device_dirs.bin_dir) sh = '%s.sh' % cmd[0] self.m.run.writefile(self.m.vars.tmp_dir.join(sh), - 'set -x; %s%s; echo $? >%src' % - (self.m.vars.android_bin_dir, subprocess.list2cmdline(map(str, cmd)), - self.m.vars.android_bin_dir)) + 'set -x; %s%s; echo $? >%src' % ( + self.device_dirs.bin_dir, subprocess.list2cmdline(map(str, cmd)), + self.device_dirs.bin_dir)) self._adb('push %s' % sh, - 'push', self.m.vars.tmp_dir.join(sh), self.m.vars.android_bin_dir) + 'push', self.m.vars.tmp_dir.join(sh), self.device_dirs.bin_dir) self._adb('clear log', 'logcat', '-c') self.m.python.inline('%s' % cmd[0], """ @@ -479,7 +482,7 @@ wait_for_device() print "Couldn't read the return code. Probably killed for OOM." sys.exit(1) """ % (self.ADB_BINARY, self.ADB_BINARY), - args=[self.m.vars.android_bin_dir, sh]) + args=[self.device_dirs.bin_dir, sh]) def copy_file_to_device(self, host, device): self._adb('push %s %s' % (host, device), 'push', host, device) diff --git a/infra/bots/recipe_modules/flavor/gn_chromebook_flavor.py b/infra/bots/recipe_modules/flavor/gn_chromebook_flavor.py index 984eccef25..a133f2bb90 100644 --- a/infra/bots/recipe_modules/flavor/gn_chromebook_flavor.py +++ b/infra/bots/recipe_modules/flavor/gn_chromebook_flavor.py @@ -21,16 +21,16 @@ class GNChromebookFlavorUtils(gn_flavor.GNFlavorUtils): super(GNChromebookFlavorUtils, self).__init__(m) self._user_ip = '' + self.chromeos_homedir = '/home/chronos/user/' self.device_dirs = default_flavor.DeviceDirs( - dm_dir = self.m.vars.chromeos_homedir + 'dm_out', - perf_data_dir = self.m.vars.chromeos_homedir + 'perf', - resource_dir = self.m.vars.chromeos_homedir + 'resources', - images_dir = self.m.vars.chromeos_homedir + 'images', - skp_dir = self.m.vars.chromeos_homedir + 'skps', - svg_dir = self.m.vars.chromeos_homedir + 'svgs', - tmp_dir = self.m.vars.chromeos_homedir) - - self._bin_dir = self.m.vars.chromeos_homedir + 'bin' + bin_dir = self.chromeos_homedir + 'bin', + dm_dir = self.chromeos_homedir + 'dm_out', + perf_data_dir = self.chromeos_homedir + 'perf', + resource_dir = self.chromeos_homedir + 'resources', + images_dir = self.chromeos_homedir + 'images', + skp_dir = self.chromeos_homedir + 'skps', + svg_dir = self.chromeos_homedir + 'svgs', + tmp_dir = self.chromeos_homedir) @property def user_ip(self): @@ -62,10 +62,10 @@ class GNChromebookFlavorUtils(gn_flavor.GNFlavorUtils): self.device_dirs.resource_dir) # Ensure the home dir is marked executable - self._ssh('remount %s as exec' % self.m.vars.chromeos_homedir, + self._ssh('remount %s as exec' % self.chromeos_homedir, 'sudo', 'mount', '-i', '-o', 'remount,exec', '/home/chronos') - self.create_clean_device_dir(self._bin_dir) + self.create_clean_device_dir(self.device_dirs.bin_dir) def create_clean_device_dir(self, path): # use -f to silently return if path doesn't exist @@ -121,13 +121,13 @@ class GNChromebookFlavorUtils(gn_flavor.GNFlavorUtils): name = cmd[0] if name == 'dm': - self.create_clean_host_dir(self.m.vars.dm_dir) + self.create_clean_host_dir(self.host_dirs.dm_dir) if name == 'nanobench': - self.create_clean_host_dir(self.m.vars.perf_data_dir) + self.create_clean_host_dir(self.host_dirs.perf_data_dir) - app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0]) + app = self.m.vars.skia_out.join(cmd[0]) - cmd[0] = '%s/%s' % (self._bin_dir, cmd[0]) + cmd[0] = '%s/%s' % (self.device_dirs.bin_dir, cmd[0]) self.copy_file_to_device(app, cmd[0]) self._ssh('chmod %s' % name, 'chmod', '+x', cmd[0]) diff --git a/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py b/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py index 9ab77eaf35..bd65e01516 100644 --- a/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py +++ b/infra/bots/recipe_modules/flavor/gn_chromecast_flavor.py @@ -15,22 +15,21 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils): super(GNChromecastFlavorUtils, self).__init__(m) self._ever_ran_adb = False self._user_ip = '' - self.m.vars.android_bin_dir = self.m.path.join(self.m.vars.android_bin_dir, - 'bin') # Disk space is extremely tight on the Chromecasts (~100M) There is not # enough space on the android_data_dir (/cache/skia) to fit the images, # resources, executable and output the dm images. So, we have dm_out be # on the tempfs (i.e. RAM) /dev/shm. (which is about 140M) - + data_dir = '/cache/skia/' self.device_dirs = default_flavor.DeviceDirs( + bin_dir = '/cache/skia/bin', dm_dir = '/dev/shm/skia/dm_out', - perf_data_dir = self.m.vars.android_data_dir + 'perf', - resource_dir = self.m.vars.android_data_dir + 'resources', - images_dir = self.m.vars.android_data_dir + 'images', - skp_dir = self.m.vars.android_data_dir + 'skps', - svg_dir = self.m.vars.android_data_dir + 'svgs', - tmp_dir = self.m.vars.android_data_dir) + perf_data_dir = data_dir + 'perf', + resource_dir = data_dir + 'resources', + images_dir = data_dir + 'images', + skp_dir = data_dir + 'skps', + svg_dir = data_dir + 'svgs', + tmp_dir = data_dir) @property def user_ip_host(self): @@ -53,8 +52,8 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils): def install(self): super(GNChromecastFlavorUtils, self).install() - self._adb('mkdir ' + self.m.vars.android_bin_dir, - 'shell', 'mkdir', '-p', self.m.vars.android_bin_dir) + self._adb('mkdir ' + self.device_dirs.bin_dir, + 'shell', 'mkdir', '-p', self.device_dirs.bin_dir) def _adb(self, title, *cmd, **kwargs): if not self._ever_ran_adb: @@ -104,7 +103,7 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils): def cleanup_steps(self): if self._ever_ran_adb: # To clean up disk space for next time - self._ssh('Delete executables', 'rm', '-r', self.m.vars.android_bin_dir, + self._ssh('Delete executables', 'rm', '-r', self.device_dirs.bin_dir, abort_on_failure=False, infra_step=True) # Reconnect if was disconnected self._adb('disconnect', 'disconnect') @@ -125,7 +124,7 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils): line = line.replace(addr, addr + ' ' + sym.strip()) print line """, - args=[self.m.vars.skia_out.join(self.m.vars.configuration)], + args=[self.m.vars.skia_out], infra_step=True, abort_on_failure=False) @@ -145,10 +144,10 @@ class GNChromecastFlavorUtils(gn_android_flavor.GNAndroidFlavorUtils): return self.m.run(self.m.step, title, cmd=ssh_cmd, **kwargs) def step(self, name, cmd, **kwargs): - app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0]) + app = self.m.vars.skia_out.join(cmd[0]) self._adb('push %s' % cmd[0], - 'push', app, self.m.vars.android_bin_dir) + 'push', app, self.device_dirs.bin_dir) - cmd[0] = '%s/%s' % (self.m.vars.android_bin_dir, cmd[0]) + cmd[0] = '%s/%s' % (self.device_dirs.bin_dir, cmd[0]) self._ssh(str(name), *cmd, infra_step=False) diff --git a/infra/bots/recipe_modules/flavor/gn_flavor.py b/infra/bots/recipe_modules/flavor/gn_flavor.py index 8d16015ca4..284f20b365 100644 --- a/infra/bots/recipe_modules/flavor/gn_flavor.py +++ b/infra/bots/recipe_modules/flavor/gn_flavor.py @@ -16,7 +16,7 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): infra_step=infra_step) def step(self, name, cmd): - app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0]) + app = self.device_dirs.bin_dir.join(cmd[0]) cmd = [app] + cmd[1:] env = self.m.context.env path = [] @@ -43,7 +43,8 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): ld_library_path.append(slave_dir.join('linux_vulkan_sdk', 'lib')) if 'SwiftShader' in extra_tokens: - ld_library_path.append(self.m.vars.skia_out.join('swiftshader_out')) + ld_library_path.append( + self.m.vars.build_dir.join('out', 'swiftshader_out')) if 'MSAN' in extra_tokens: # Find the MSAN-built libc++. @@ -60,7 +61,8 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): elif self.m.vars.is_linux: cmd = ['catchsegv'] + cmd elif 'ProcDump' in extra_tokens: - self.m.file.ensure_directory('makedirs dumps', self.m.vars.dumps_dir) + dumps_dir = self.m.path.join(self.m.vars.swarming_out_dir, 'dumps') + self.m.file.ensure_directory('makedirs dumps', dumps_dir) procdump = str(self.m.vars.slave_dir.join('procdump_win', 'procdump64.exe')) # Full docs for ProcDump here: @@ -70,8 +72,7 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): # -e 1 tells procdump to dump once # -x <dump dir> <exe> <args> launches exe and writes dumps to the # specified dir - cmd = [procdump, '-accepteula', '-mp', '-e', '1', - '-x', self.m.vars.dumps_dir] + cmd + cmd = [procdump, '-accepteula', '-mp', '-e', '1', '-x', dumps_dir] + cmd if 'ASAN' in extra_tokens or 'UBSAN' in extra_tokens: if 'Mac' in self.m.vars.builder_cfg.get('os', ''): @@ -104,7 +105,7 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): # Convert path objects or placeholders into strings such that they can # be passed to symbolize_stack_trace.py args = [slave_dir] + [str(x) for x in cmd] - with self.m.context(cwd=self.m.vars.skia_dir, env=env): + with self.m.context(cwd=self.m.path['start_dir'].join('skia'), env=env): self._py('symbolized %s' % name, self.module.resource('symbolize_stack_trace.py'), args=args, diff --git a/infra/bots/recipe_modules/flavor/ios_flavor.py b/infra/bots/recipe_modules/flavor/ios_flavor.py index fb4b8a8adb..3df07451e9 100644 --- a/infra/bots/recipe_modules/flavor/ios_flavor.py +++ b/infra/bots/recipe_modules/flavor/ios_flavor.py @@ -10,6 +10,17 @@ import gn_flavor import os class iOSFlavorUtils(gn_flavor.GNFlavorUtils): + def __init__(self, m): + super(iOSFlavorUtils, self).__init__(m) + self.device_dirs = default_flavor.DeviceDirs( + bin_dir='[unused]', + dm_dir='dm', + perf_data_dir='perf', + resource_dir='resources', + images_dir='images', + skp_dir='skps', + svg_dir='svgs', + tmp_dir='tmp') def install(self): # Set up the device @@ -17,8 +28,7 @@ class iOSFlavorUtils(gn_flavor.GNFlavorUtils): # Install the app. for app_name in ['dm', 'nanobench']: - app_package = self.m.vars.skia_out.join(self.m.vars.configuration, - '%s.app' % app_name) + app_package = self.m.vars.skia_out.join('%s.app' % app_name) def uninstall_app(attempt): # If app ID changes, upgrade will fail, so try uninstalling. @@ -35,15 +45,6 @@ class iOSFlavorUtils(gn_flavor.GNFlavorUtils): between_attempts_fn=uninstall_app, infra_step=True) - self.device_dirs = default_flavor.DeviceDirs( - dm_dir='dm', - perf_data_dir='perf', - resource_dir='resources', - images_dir='images', - skp_dir='skps', - svg_dir='svgs', - tmp_dir='tmp') - def step(self, name, cmd, env=None, **kwargs): bundle_id = 'com.google.%s' % cmd[0] self.m.run(self.m.step, name, @@ -51,7 +52,8 @@ class iOSFlavorUtils(gn_flavor.GNFlavorUtils): map(str, cmd[1:])) def _run_ios_script(self, script, first, *rest): - full = self.m.vars.skia_dir.join('platform_tools/ios/bin/ios_' + script) + full = self.m.path['start_dir'].join( + 'skia', 'platform_tools', 'ios', 'bin', 'ios_' + script) self.m.run(self.m.step, name = '%s %s' % (script, first), cmd = [full, first] + list(rest), @@ -74,7 +76,8 @@ class iOSFlavorUtils(gn_flavor.GNFlavorUtils): self._run_ios_script('mkdir', path) def read_file_on_device(self, path, **kwargs): - full = self.m.vars.skia_dir.join('platform_tools/ios/bin/ios_cat_file') + full = self.m.path['start_dir'].join( + 'skia', 'platform_tools', 'ios', 'bin', 'ios_cat_file') rv = self.m.run(self.m.step, name = 'cat_file %s' % path, cmd = [full, path], diff --git a/infra/bots/recipe_modules/flavor/valgrind_flavor.py b/infra/bots/recipe_modules/flavor/valgrind_flavor.py index 4e3475b19f..b54fcc36dd 100644 --- a/infra/bots/recipe_modules/flavor/valgrind_flavor.py +++ b/infra/bots/recipe_modules/flavor/valgrind_flavor.py @@ -12,8 +12,8 @@ import gn_flavor class ValgrindFlavorUtils(gn_flavor.GNFlavorUtils): def __init__(self, m): super(ValgrindFlavorUtils, self).__init__(m) - self._suppressions_file = self.m.vars.skia_dir.join( - 'tools', 'valgrind.supp') + self._suppressions_file = self.m.path['start_dir'].join( + 'skia', 'tools', 'valgrind.supp') self._valgrind_cipd_dir = self.m.vars.slave_dir.join('valgrind') self._valgrind_fake_dir = self._valgrind_cipd_dir self._valgrind = self._valgrind_fake_dir.join('bin', 'valgrind') @@ -23,7 +23,7 @@ class ValgrindFlavorUtils(gn_flavor.GNFlavorUtils): new_cmd = [self._valgrind, '--gen-suppressions=all', '--leak-check=full', '--track-origins=yes', '--error-exitcode=1', '--num-callers=40', '--suppressions=%s' % self._suppressions_file] - path_to_app = self.m.vars.skia_out.join(self.m.vars.configuration, cmd[0]) + path_to_app = self.m.vars.skia_out.join(cmd[0]) new_cmd.append(path_to_app) new_cmd.extend(cmd[1:]) with self.m.env({'VALGRIND_LIB': self._lib_dir}): diff --git a/infra/bots/recipe_modules/run/api.py b/infra/bots/recipe_modules/run/api.py index 3464ef7bbf..20dd6c7b85 100644 --- a/infra/bots/recipe_modules/run/api.py +++ b/infra/bots/recipe_modules/run/api.py @@ -51,14 +51,14 @@ class SkiaStepApi(recipe_api.RecipeApi): """Wrapper around api.file.rmtree.""" self.m.file.rmtree('rmtree %s' % self.m.path.basename(path), path) - def asset_version(self, asset_name, test_data=None): + def asset_version(self, asset_name, skia_dir, test_data=None): """Return the contents of VERSION for the given asset as a string. If test_data is not specified, reads the property 'test_<asset_name>_version' or if not present, uses TEST_DEFAULT_ASSET_VERSION.""" - version_file = self.m.vars.infrabots_dir.join( - 'assets', asset_name, 'VERSION') + version_file = skia_dir.join( + 'infra', 'bots', 'assets', asset_name, 'VERSION') if not test_data: test_data = self.m.properties.get( 'test_%s_version' % asset_name, TEST_DEFAULT_ASSET_VERSION) diff --git a/infra/bots/recipe_modules/run/examples/full.py b/infra/bots/recipe_modules/run/examples/full.py index 7f93233514..d6a27c9b96 100644 --- a/infra/bots/recipe_modules/run/examples/full.py +++ b/infra/bots/recipe_modules/run/examples/full.py @@ -35,7 +35,7 @@ def RunSteps(api): api.run.readfile('myfile.txt') api.run.writefile('myfile.txt', 'contents') api.run.rmtree('mydir') - api.run.asset_version('my_asset') + api.run.asset_version('my_asset', api.vars.cache_dir.join('work', 'skia')) # Merge PATHs. with api.context(env={'PATH': 'mydir:%(PATH)s'}): diff --git a/infra/bots/recipe_modules/vars/api.py b/infra/bots/recipe_modules/vars/api.py index e7471089b3..44845a41e8 100644 --- a/infra/bots/recipe_modules/vars/api.py +++ b/infra/bots/recipe_modules/vars/api.py @@ -15,9 +15,6 @@ CONFIG_RELEASE = 'Release' class SkiaVarsApi(recipe_api.RecipeApi): - override_checkout_root = None - override_gclient_cache = None - def setup(self): """Prepare the variables.""" # Setup @@ -27,87 +24,19 @@ class SkiaVarsApi(recipe_api.RecipeApi): # Special input/output directories. self.build_dir = self.slave_dir.join('build') - self.test_dir = self.slave_dir.join('test') - self.perf_dir = self.slave_dir.join('perf') - self.checkout_root = self.slave_dir self.default_env = self.m.context.env self.default_env['CHROME_HEADLESS'] = '1' self.default_env['PATH'] = self.m.path.pathsep.join([ self.default_env.get('PATH', '%(PATH)s'), str(self.m.bot_update._module.PACKAGE_REPO_ROOT), ]) - self.gclient_env = {'DEPOT_TOOLS_UPDATE': '0'} - self.is_compile_bot = self.builder_name.startswith('Build-') - - self.persistent_checkout = False - # Compile bots keep a persistent checkout. - if self.is_compile_bot and 'NoDEPS' not in self.builder_name: - self.persistent_checkout = True - if 'Housekeeper' in self.builder_name: - self.persistent_checkout = True - if '-CT_' in self.builder_name: - self.persistent_checkout = True - # We need the source code for the Coverage's Upload step to be in the - # same absolute location as when we compiled it so we can map the - # coverage data to actual line numbers. We ensure this by making sure - # we have a checkout on the Coverage's Upload step and that the Upload - # step runs on the same bots that Compile. - if 'Coverage' in self.builder_name and 'Upload' in self.builder_name: - self.persistent_checkout = True - self.cache_dir = self.slave_dir.join('cache') - if self.persistent_checkout: - self.checkout_root = self.cache_dir.join('work') - self.gclient_cache = self.cache_dir.join('git') - if self.override_checkout_root: - self.checkout_root = self.override_checkout_root - self.gclient_cache = self.override_gclient_cache - # got_revision is filled in after checkout steps. - self.got_revision = None - else: - # If there's no persistent checkout, then we have to assume we got the - # correct revision of the files from isolate. - self.got_revision = self.m.properties['revision'] - - # Some bots also require a checkout of Flutter; in this case we use the - # checkout of Skia obtained through DEPS in src/third_party/skia. - self.need_flutter_checkout = 'Flutter' in self.builder_name - - self.skia_dir = self.checkout_root.join('skia') - if self.need_flutter_checkout: - self.checkout_root = self.checkout_root.join('flutter') - self.skia_dir = self.checkout_root.join('src', 'third_party', 'skia') - - if not self.persistent_checkout: - self.m.path['checkout'] = self.skia_dir - - self.infrabots_dir = self.skia_dir.join('infra', 'bots') - self.resource_dir = self.skia_dir.join('resources') - self.images_dir = self.slave_dir.join('skimage') - self.skia_out = self.skia_dir.join('out', self.builder_name) + self.swarming_out_dir = self.slave_dir.join( self.m.properties['swarm_out_dir']) - if 'ParentRevision' in self.builder_name: - # Tasks that depend on ParentRevision builds usually also depend on a - # second build task. Use a different path for build results so that the - # binaries end up in different directories in the isolate. - self.swarming_out_dir = self.swarming_out_dir.join('ParentRevision') - self.local_skp_dir = self.slave_dir.join('skp') - self.local_svg_dir = self.slave_dir.join('svg') - if not self.is_compile_bot: - self.skia_out = self.build_dir.join('out') - self.tmp_dir = self.m.path['start_dir'].join('tmp') - # Some bots also require a checkout of chromium. - self.need_chromium_checkout = False - if 'CommandBuffer' in self.builder_name: - self.need_chromium_checkout = True - self.gclient_env['GYP_CHROMIUM_NO_ACTION'] = '0' - if 'RecreateSKPs' in self.builder_name: - self.need_chromium_checkout = True - self.gclient_env['CPPFLAGS'] = ( - '-DSK_ALLOW_CROSSPROCESS_PICTUREIMAGEFILTERS=1') + self.tmp_dir = self.m.path['start_dir'].join('tmp') self.builder_cfg = self.m.builder_name_schema.DictForBuilderName( self.builder_name) @@ -120,6 +49,9 @@ class SkiaVarsApi(recipe_api.RecipeApi): arch = (self.builder_cfg.get('arch') or self.builder_cfg.get('target_arch')) if ('Win' in self.builder_cfg.get('os', '') and arch == 'x86_64'): self.configuration += '_x64' + + self.skia_out = self.build_dir.join('out', self.configuration) + self.extra_tokens = [] if len(self.builder_cfg.get('extra_config', '')) > 0: if self.builder_cfg['extra_config'].startswith('SK'): @@ -138,27 +70,9 @@ class SkiaVarsApi(recipe_api.RecipeApi): self.issue = self.m.properties['patch_issue'] self.patchset = self.m.properties['patch_set'] - self.dm_dir = self.m.path.join( - self.swarming_out_dir, 'dm') - self.perf_data_dir = self.m.path.join(self.swarming_out_dir, - 'perfdata', self.builder_name, 'data') - self.dumps_dir = self.m.path.join( - self.swarming_out_dir, 'dumps') self._swarming_bot_id = None self._swarming_task_id = None - # Data should go under in _data_dir, which may be preserved across runs. - self.android_data_dir = '/sdcard/revenge_of_the_skiabot/' - # Executables go under _bin_dir, which, well, allows executable files. - self.android_bin_dir = '/data/local/tmp/' - - if self.builder_cfg.get('os', '') == 'Chromecast': - # On the Chromecast, everything goes in the (~110M) /cache/skia - self.android_bin_dir = '/cache/skia/' - self.android_data_dir = '/cache/skia/' - - self.chromeos_homedir = '/home/chronos/user/' - # Internal bot support. self.internal_hardware_label = ( self.m.properties.get('internal_hardware_label')) @@ -169,44 +83,6 @@ class SkiaVarsApi(recipe_api.RecipeApi): return 'Ubuntu' in self.builder_name or 'Debian' in self.builder_name @property - def upload_dm_results(self): - # TODO(borenet): Move this into the swarm_test recipe. - skip_upload_bots = [ - 'ASAN', - 'Coverage', - 'MSAN', - 'TSAN', - 'UBSAN', - 'Valgrind', - ] - upload_dm_results = True - for s in skip_upload_bots: - if s in self.m.properties['buildername']: - upload_dm_results = False - break - return upload_dm_results - - @property - def upload_perf_results(self): - # TODO(borenet): Move this into the swarm_perf recipe. - if 'Release' not in self.m.properties['buildername']: - return False - skip_upload_bots = [ - 'ASAN', - 'Coverage', - 'MSAN', - 'TSAN', - 'UBSAN', - 'Valgrind', - ] - upload_perf_results = True - for s in skip_upload_bots: - if s in self.m.properties['buildername']: - upload_perf_results = False - break - return upload_perf_results - - @property def swarming_bot_id(self): if not self._swarming_bot_id: self._swarming_bot_id = self.m.python.inline( diff --git a/infra/bots/recipe_modules/vars/examples/full.expected/Build-Debian9-Clang-x86_64-Release-ParentRevision.json b/infra/bots/recipe_modules/vars/examples/full.expected/Build-Debian9-Clang-x86_64-Release-ParentRevision.json deleted file mode 100644 index 4594f9e6b9..0000000000 --- a/infra/bots/recipe_modules/vars/examples/full.expected/Build-Debian9-Clang-x86_64-Release-ParentRevision.json +++ /dev/null @@ -1,35 +0,0 @@ -[ - { - "cmd": [ - "python", - "-u", - "import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n" - ], - "name": "get swarming bot id", - "stdout": "/path/to/tmp/", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@import os@@@", - "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", - "import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n" - ], - "name": "get swarming task id", - "stdout": "/path/to/tmp/", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@import os@@@", - "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "name": "$result", - "recipe_result": null, - "status_code": 0 - } -]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/vars/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json b/infra/bots/recipe_modules/vars/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json deleted file mode 100644 index 4594f9e6b9..0000000000 --- a/infra/bots/recipe_modules/vars/examples/full.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json +++ /dev/null @@ -1,35 +0,0 @@ -[ - { - "cmd": [ - "python", - "-u", - "import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n" - ], - "name": "get swarming bot id", - "stdout": "/path/to/tmp/", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@import os@@@", - "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", - "import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n" - ], - "name": "get swarming task id", - "stdout": "/path/to/tmp/", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@import os@@@", - "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "name": "$result", - "recipe_result": null, - "status_code": 0 - } -]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/vars/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json b/infra/bots/recipe_modules/vars/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json deleted file mode 100644 index 4594f9e6b9..0000000000 --- a/infra/bots/recipe_modules/vars/examples/full.expected/Build-Mac-Clang-x86_64-Debug-CommandBuffer.json +++ /dev/null @@ -1,35 +0,0 @@ -[ - { - "cmd": [ - "python", - "-u", - "import os\nprint os.environ.get('SWARMING_BOT_ID', '')\n" - ], - "name": "get swarming bot id", - "stdout": "/path/to/tmp/", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@import os@@@", - "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_BOT_ID', '')@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "cmd": [ - "python", - "-u", - "import os\nprint os.environ.get('SWARMING_TASK_ID', '')\n" - ], - "name": "get swarming task id", - "stdout": "/path/to/tmp/", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@python.inline@import os@@@", - "@@@STEP_LOG_LINE@python.inline@print os.environ.get('SWARMING_TASK_ID', '')@@@", - "@@@STEP_LOG_END@python.inline@@@" - ] - }, - { - "name": "$result", - "recipe_result": null, - "status_code": 0 - } -]
\ No newline at end of file diff --git a/infra/bots/recipe_modules/vars/examples/full.py b/infra/bots/recipe_modules/vars/examples/full.py index 29c16c2f71..b32667a339 100644 --- a/infra/bots/recipe_modules/vars/examples/full.py +++ b/infra/bots/recipe_modules/vars/examples/full.py @@ -16,22 +16,17 @@ def RunSteps(api): api.vars.override_gclient_cache = api.path['start_dir'] api.vars.setup() info = [ - api.vars.upload_dm_results, - api.vars.upload_perf_results, api.vars.swarming_bot_id, api.vars.swarming_task_id, ] if api.vars.is_linux: - assert len(info) == 4 # Make pylint happy. + assert len(info) == 2 # Make pylint happy. TEST_BUILDERS = [ 'Build-Debian9-Clang-x86_64-Release-NoDEPS', 'Build-Debian9-Clang-x86_64-Release-Override', - 'Build-Debian9-Clang-x86_64-Release-ParentRevision', 'Build-Debian9-Clang-x86_64-Release-SKNX_NO_SIMD', - 'Build-Debian9-GCC-x86_64-Release-Flutter_Android', - 'Build-Mac-Clang-x86_64-Debug-CommandBuffer', 'Build-Win-Clang-x86_64-Release-Vulkan', 'Housekeeper-Weekly-RecreateSKPs', 'Perf-Chromecast-GCC-Chorizo-CPU-Cortex_A7-arm-Debug-All', diff --git a/infra/bots/recipes/bookmaker.py b/infra/bots/recipes/bookmaker.py index 7f52786ed1..f499422516 100644 --- a/infra/bots/recipes/bookmaker.py +++ b/infra/bots/recipes/bookmaker.py @@ -39,13 +39,14 @@ def go_get_fiddlecli(api): def RunSteps(api): api.vars.setup() - api.core.checkout_bot_update() + checkout_root = api.core.default_checkout_root + api.core.checkout_bot_update(checkout_root=checkout_root) api.infra.go_version() go_get_fiddlecli(api) - with api.context(cwd=api.vars.skia_dir, env=api.infra.go_env): - bookmaker_binary = api.path.join(api.vars.skia_out, api.vars.configuration, - 'bookmaker') + skia_dir = checkout_root.join('skia') + with api.context(cwd=skia_dir, env=api.infra.go_env): + bookmaker_binary = api.vars.skia_out.join('bookmaker') buildername = api.vars.builder_name if 'PerCommit' in buildername: @@ -125,10 +126,10 @@ def RunSteps(api): # Step 4: Update docs in site/user/api/ with the output of fiddlecli. # If there are any new changes then upload and commit the changes. cmd = ['python', - api.vars.skia_dir.join('infra', 'bots', 'upload_md.py'), + skia_dir.join('infra', 'bots', 'upload_md.py'), '--bookmaker_binary', bookmaker_binary, '--fiddlecli_output', fiddlecli_output] - with api.context(cwd=api.vars.skia_dir, env=api.infra.go_env): + with api.context(cwd=skia_dir, env=api.infra.go_env): api.run(api.step, 'Generate and Upload Markdown files', cmd=cmd) diff --git a/infra/bots/recipes/calmbench.py b/infra/bots/recipes/calmbench.py index 30343f2712..f75cd35d72 100644 --- a/infra/bots/recipes/calmbench.py +++ b/infra/bots/recipes/calmbench.py @@ -28,7 +28,8 @@ def RunSteps(api): api.flavor.install(skps=True, svgs=True) api.file.ensure_directory('makedirs perf', api.vars.swarming_out_dir) - with api.context(cwd=api.vars.skia_dir): + skia_dir = api.path['start_dir'].join('skia') + with api.context(cwd=skia_dir): extra_arg = '--svgs %s --skps %s' % (api.flavor.device_dirs.svg_dir, api.flavor.device_dirs.skp_dir) @@ -41,7 +42,7 @@ def RunSteps(api): command = [ 'python', - api.vars.skia_dir.join('tools', 'calmbench', 'ab.py'), + skia_dir.join('tools', 'calmbench', 'ab.py'), api.vars.swarming_out_dir, 'modified', 'master', api.vars.build_dir.join("out", api.vars.configuration, 'nanobench'), @@ -53,7 +54,7 @@ def RunSteps(api): config, -1, # threads; let ab.py decide the threads "false", # noinit - "--githash", api.vars.got_revision, + "--githash", api.properties['revision'], "--concise" ] diff --git a/infra/bots/recipes/check_generated_files.py b/infra/bots/recipes/check_generated_files.py index b0c72f2ed6..61a859f26a 100644 --- a/infra/bots/recipes/check_generated_files.py +++ b/infra/bots/recipes/check_generated_files.py @@ -24,7 +24,8 @@ DEPS = [ def RunSteps(api): # Checkout, compile, etc. api.vars.setup() - api.core.checkout_bot_update() + checkout_root = api.core.default_checkout_root + api.core.checkout_bot_update(checkout_root=checkout_root) api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) api.flavor.setup() @@ -55,7 +56,8 @@ for r, d, files in os.walk('%s'): """ % cwd) # Regenerate the SKSL files. - api.build() + api.build(checkout_root=checkout_root, + out_dir=api.vars.build_dir.join('out', 'Release')) # Get a second diff. If this doesn't match the first, then there have been # modifications to the generated files. diff --git a/infra/bots/recipes/compile.expected/Build-Debian9-Clang-universal-devrel-Android_SKQP.json b/infra/bots/recipes/compile.expected/Build-Debian9-Clang-universal-devrel-Android_SKQP.json new file mode 100644 index 0000000000..8e5e8060eb --- /dev/null +++ b/infra/bots/recipes/compile.expected/Build-Debian9-Clang-universal-devrel-Android_SKQP.json @@ -0,0 +1,272 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/cache/work" + ], + "infra_step": true, + "name": "makedirs checkout_path" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "remove", + "[START_DIR]/cache/work/.gclient_entries" + ], + "infra_step": true, + "name": "remove [START_DIR]/cache/work/.gclient_entries" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec-path", + "cache_dir = '[START_DIR]/cache/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]", + "--patch_root", + "skia", + "--revision_mapping_file", + "{\"got_revision\": \"skia\"}", + "--git-cache-dir", + "[START_DIR]/cache/git", + "--cleanup-dir", + "[CLEANUP]/bot_update", + "--output_json", + "/path/to/tmp/json", + "--revision", + "skia@abc123" + ], + "cwd": "[START_DIR]/cache/work", + "env_prefixes": { + "PATH": [ + "RECIPE_PACKAGE_REPO[depot_tools]" + ] + }, + "infra_step": true, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/tmp" + ], + "infra_step": true, + "name": "makedirs tmp_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/cache/work/skia/infra/bots/assets/android_ndk_linux/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get android_ndk_linux VERSION" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/cache/work/skia/bin/fetch-gn" + ], + "cwd": "[START_DIR]/cache/work/skia", + "env": { + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "infra_step": true, + "name": "fetch-gn" + }, + { + "cmd": [ + "[START_DIR]/go/go/bin/go", + "version" + ], + "cwd": "[START_DIR]/cache/work/skia", + "env": { + "CHROME_HEADLESS": "1", + "GOPATH": "[START_DIR]/gopath", + "GOROOT": "[START_DIR]/go/go", + "PATH": "[START_DIR]/go/go/bin:[START_DIR]/gopath:<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "go version" + }, + { + "cmd": [ + "go", + "version" + ], + "cwd": "[START_DIR]/cache/work/skia", + "env": { + "CHROME_HEADLESS": "1", + "GOPATH": "[START_DIR]/gopath", + "GOROOT": "[START_DIR]/go/go", + "PATH": "[START_DIR]/go/go/bin:[START_DIR]/gopath:<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "env go version" + }, + { + "cmd": [ + "[START_DIR]/go/go/bin/go", + "get", + "-u", + "-t", + "go.skia.org/infra/..." + ], + "cwd": "[START_DIR]/cache/work/skia", + "env": { + "CHROME_HEADLESS": "1", + "GOPATH": "[START_DIR]/gopath", + "GOROOT": "[START_DIR]/go/go", + "PATH": "[START_DIR]/go/go/bin:[START_DIR]/gopath:<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "update go pkgs" + }, + { + "cmd": [ + "go", + "build", + "-o", + "[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-universal-devrel-Android_SKQP/devrel/run_testlab", + "[START_DIR]/cache/work/skia/infra/cts/run_testlab.go" + ], + "cwd": "[START_DIR]/cache/work/skia", + "env": { + "CHROME_HEADLESS": "1", + "GOPATH": "[START_DIR]/gopath", + "GOROOT": "[START_DIR]/go/go", + "PATH": "[START_DIR]/go/go/bin:[START_DIR]/gopath:<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "build firebase runner" + }, + { + "cmd": [ + "[START_DIR]/cache/work/skia/tools/skqp/make_universal_apk" + ], + "cwd": "[START_DIR]/cache/work/skia", + "env": { + "ANDROID_HOME": "[START_DIR]/android_sdk_linux/android-sdk", + "ANDROID_NDK": "[START_DIR]/android_ndk_linux", + "APK_OUTPUT_DIR": "[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-universal-devrel-Android_SKQP/devrel", + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "make_universal" + }, + { + "cmd": [ + "python", + "-u", + "import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n", + "[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-universal-devrel-Android_SKQP/devrel", + "[START_DIR]/[SWARM_OUT_DIR]/out/devrel" + ], + "infra_step": true, + "name": "copy build products", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import errno@@@", + "@@@STEP_LOG_LINE@python.inline@import glob@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import shutil@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@", + "@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@try:@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@", + "@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@", + "@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@", + "@@@STEP_LOG_LINE@python.inline@ raise@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@", + "@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@", + "@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@", + "@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@", + "@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/cache/work/skia/infra/cts/whitelist_devices.json", + "[START_DIR]/[SWARM_OUT_DIR]/out/devrel" + ], + "infra_step": true, + "name": "copy whitelist" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipes/compile.expected/Build-Debian9-Clang-x86_64-Release-ParentRevision.json b/infra/bots/recipes/compile.expected/Build-Debian9-Clang-x86_64-Release-ParentRevision.json new file mode 100644 index 0000000000..975db93e65 --- /dev/null +++ b/infra/bots/recipes/compile.expected/Build-Debian9-Clang-x86_64-Release-ParentRevision.json @@ -0,0 +1,211 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/cache/work" + ], + "infra_step": true, + "name": "makedirs checkout_path" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "remove", + "[START_DIR]/cache/work/.gclient_entries" + ], + "infra_step": true, + "name": "remove [START_DIR]/cache/work/.gclient_entries" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py", + "--spec-path", + "cache_dir = '[START_DIR]/cache/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'skia', 'url': 'https://skia.googlesource.com/skia.git'}]", + "--patch_root", + "skia", + "--revision_mapping_file", + "{\"got_revision\": \"skia\"}", + "--git-cache-dir", + "[START_DIR]/cache/git", + "--cleanup-dir", + "[CLEANUP]/bot_update", + "--output_json", + "/path/to/tmp/json", + "--revision", + "skia@abc123^" + ], + "cwd": "[START_DIR]/cache/work", + "env_prefixes": { + "PATH": [ + "RECIPE_PACKAGE_REPO[depot_tools]" + ] + }, + "infra_step": true, + "name": "bot_update", + "~followup_annotations": [ + "@@@STEP_TEXT@Some step text@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@", + "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": \"abc123^\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@", + "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"got_revision_cp\": \"refs/heads/master@{#164710}\"@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"root\": \"skia\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"skia\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/skia.git\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@", + "@@@STEP_LOG_LINE@json.output@ }, @@@", + "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@", + "@@@SET_BUILD_PROPERTY@got_revision@\"9046e2e693bb92a76e972b694580e5d17ad10748\"@@@", + "@@@SET_BUILD_PROPERTY@got_revision_cp@\"refs/heads/master@{#164710}\"@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/tmp" + ], + "infra_step": true, + "name": "makedirs tmp_dir" + }, + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/cache/work/skia/infra/bots/assets/clang_linux/VERSION", + "/path/to/tmp/" + ], + "infra_step": true, + "name": "Get clang_linux VERSION" + }, + { + "cmd": [ + "python", + "-u", + "[START_DIR]/cache/work/skia/bin/fetch-gn" + ], + "cwd": "[START_DIR]/cache/work/skia", + "env": { + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "infra_step": true, + "name": "fetch-gn" + }, + { + "cmd": [ + "[START_DIR]/cache/work/skia/bin/gn", + "gen", + "[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-x86_64-Release-ParentRevision/Release", + "--args=cc=\"[START_DIR]/clang_linux/bin/clang\" cxx=\"[START_DIR]/clang_linux/bin/clang++\" extra_cflags=[\"-B[START_DIR]/clang_linux/bin\", \"-DDUMMY_clang_linux_version=42\"] extra_ldflags=[\"-B[START_DIR]/clang_linux/bin\", \"-fuse-ld=lld\"] is_debug=false target_cpu=\"x86_64\"" + ], + "cwd": "[START_DIR]/cache/work/skia", + "env": { + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "gn gen" + }, + { + "cmd": [ + "ninja", + "-k", + "0", + "-C", + "[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-x86_64-Release-ParentRevision/Release" + ], + "cwd": "[START_DIR]/cache/work/skia", + "env": { + "CHROME_HEADLESS": "1", + "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]" + }, + "name": "ninja" + }, + { + "cmd": [ + "python", + "-u", + "import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n", + "[START_DIR]/cache/work/skia/out/Build-Debian9-Clang-x86_64-Release-ParentRevision/Release", + "[START_DIR]/[SWARM_OUT_DIR]/ParentRevision/out/Release" + ], + "infra_step": true, + "name": "copy build products", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@import errno@@@", + "@@@STEP_LOG_LINE@python.inline@import glob@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import shutil@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@", + "@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@", + "@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@try:@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@", + "@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@", + "@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@", + "@@@STEP_LOG_LINE@python.inline@ raise@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@", + "@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@", + "@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@", + "@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@", + "@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@", + "@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@", + "@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@", + "@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +]
\ No newline at end of file diff --git a/infra/bots/recipes/compile.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json b/infra/bots/recipes/compile.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json index 9eee4a56cd..d7d5ebafb0 100644 --- a/infra/bots/recipes/compile.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json +++ b/infra/bots/recipes/compile.expected/Build-Debian9-GCC-x86_64-Release-Flutter_Android.json @@ -170,7 +170,7 @@ "cmd": [ "ninja", "-C", - "out/android_release", + "[START_DIR]/cache/work/flutter/src/out/android_release", "-j100" ], "cwd": "[START_DIR]/cache/work/flutter/src", @@ -185,7 +185,7 @@ "python", "-u", "import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n", - "[START_DIR]/cache/work/flutter/src/third_party/skia/out/Build-Debian9-GCC-x86_64-Release-Flutter_Android/Release", + "[START_DIR]/cache/work/flutter/src/out/android_release", "[START_DIR]/[SWARM_OUT_DIR]/out/Release" ], "infra_step": true, diff --git a/infra/bots/recipes/compile.expected/flutter_trybot.json b/infra/bots/recipes/compile.expected/flutter_trybot.json index 74ab4c1ae9..113adf2f84 100644 --- a/infra/bots/recipes/compile.expected/flutter_trybot.json +++ b/infra/bots/recipes/compile.expected/flutter_trybot.json @@ -174,7 +174,7 @@ "cmd": [ "ninja", "-C", - "out/android_release", + "[START_DIR]/cache/work/flutter/src/out/android_release", "-j100" ], "cwd": "[START_DIR]/cache/work/flutter/src", @@ -189,7 +189,7 @@ "python", "-u", "import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'nanobench', 'nanobench.exe', 'skpbench', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n", - "[START_DIR]/cache/work/flutter/src/third_party/skia/out/Build-Debian9-GCC-x86_64-Release-Flutter_Android/Release", + "[START_DIR]/cache/work/flutter/src/out/android_release", "[START_DIR]/[SWARM_OUT_DIR]/out/Release" ], "infra_step": true, diff --git a/infra/bots/recipes/compile.py b/infra/bots/recipes/compile.py index b05f97134f..5d35d78af7 100644 --- a/infra/bots/recipes/compile.py +++ b/infra/bots/recipes/compile.py @@ -27,16 +27,35 @@ def RunSteps(api): # Check out code. if 'NoDEPS' in api.properties['buildername']: - api.core.checkout_git() + checkout_root = api.path['start_dir'] + api.core.checkout_git(checkout_root=checkout_root) else: - api.core.checkout_bot_update() + checkout_root = api.core.default_checkout_root + if 'Flutter' in api.vars.builder_name: + checkout_root = checkout_root.join('flutter') + api.core.checkout_bot_update(checkout_root=checkout_root) + api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) + out_dir = checkout_root.join( + 'skia', 'out', api.vars.builder_name, api.vars.configuration) + if 'Flutter' in api.vars.builder_name: + out_dir = checkout_root.join('src', 'out', 'android_release') + try: - api.build() - api.build.copy_build_products( - api.vars.swarming_out_dir.join( - 'out', api.vars.configuration)) + api.build(checkout_root=checkout_root, out_dir=out_dir) + + # TODO(borenet): Move this out of the try/finally. + dst = api.vars.swarming_out_dir.join('out', api.vars.configuration) + if 'ParentRevision' in api.vars.builder_name: + dst = api.vars.swarming_out_dir.join( + 'ParentRevision', 'out', api.vars.configuration) + api.build.copy_build_products(out_dir=out_dir, dst=dst) + if 'SKQP' in api.vars.extra_tokens: + wlist = checkout_root.join( + 'skia', 'infra','cts', 'whitelist_devices.json') + api.file.copy('copy whitelist', wlist, dst) + finally: if 'Win' in api.vars.builder_cfg.get('os', ''): api.python.inline( @@ -57,8 +76,9 @@ for p in psutil.process_iter(): TEST_BUILDERS = [ 'Build-Debian9-Clang-arm-Release-Chromebook_GLES', 'Build-Debian9-Clang-arm64-Release-Android', - 'Build-Debian9-Clang-arm64-Release-Android_Vulkan', 'Build-Debian9-Clang-arm64-Release-Android_ASAN', + 'Build-Debian9-Clang-arm64-Release-Android_Vulkan', + 'Build-Debian9-Clang-universal-devrel-Android_SKQP', 'Build-Debian9-Clang-x86_64-Debug', 'Build-Debian9-Clang-x86_64-Debug-ASAN', 'Build-Debian9-Clang-x86_64-Debug-Coverage', @@ -68,6 +88,7 @@ TEST_BUILDERS = [ 'Build-Debian9-Clang-x86_64-Release-Fast', 'Build-Debian9-Clang-x86_64-Release-Mini', 'Build-Debian9-Clang-x86_64-Release-NoDEPS', + 'Build-Debian9-Clang-x86_64-Release-ParentRevision', 'Build-Debian9-Clang-x86_64-Release-Vulkan', 'Build-Debian9-Clang-x86_64-Release-Vulkan_Coverage', 'Build-Debian9-EMCC-wasm-Release', diff --git a/infra/bots/recipes/ct_skps.py b/infra/bots/recipes/ct_skps.py index ae70e7fc72..5a679d30b5 100644 --- a/infra/bots/recipes/ct_skps.py +++ b/infra/bots/recipes/ct_skps.py @@ -75,21 +75,22 @@ def RunSteps(api): else: raise Exception('Do not recognise the buildername %s.' % buildername) - api.vars.override_checkout_root = make_path(api, '/', 'b', 'work') - api.vars.override_gclient_cache = make_path(api, '/', 'b', 'cache') - api.vars.setup() - api.core.checkout_bot_update() + checkout_root = make_path(api, '/', 'b', 'work') + gclient_cache = make_path(api, '/', 'b', 'cache') + got_revision = api.core.checkout_bot_update(checkout_root=checkout_root, + gclient_cache=gclient_cache) api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) - api.build() + out_dir = api.vars.build_dir.join('out', api.vars.configuration) + api.build(checkout_root=checkout_root, out_dir=out_dir) # Required paths. - infrabots_dir = api.vars.skia_dir.join('infra', 'bots') + infrabots_dir = checkout_root.join('skia', 'infra', 'bots') isolate_dir = infrabots_dir.join('ct') isolate_path = isolate_dir.join(CT_SKPS_ISOLATE) - api.build.copy_build_products(isolate_dir) + api.build.copy_build_products(out_dir=out_dir, dst=isolate_dir) api.skia_swarming.setup( infrabots_dir.join('tools', 'luci-go'), swarming_rev='') @@ -131,8 +132,8 @@ def RunSteps(api): # referenced also needs to change. As of 8/8/17 the other places are: # * infra/bots/ct/ct_skps.isolate # * infra/bots/ct/run_ct_skps.py - skps_dir = api.vars.checkout_root.join('skps', skps_chromium_build, - ct_page_type, str(ct_num_slaves)) + skps_dir = checkout_root.join('skps', skps_chromium_build, + ct_page_type, str(ct_num_slaves)) version_file = skps_dir.join(SKPS_VERSION_FILE) if api.path.exists(version_file): # pragma: nocover version_file_contents = api.file.read_text( @@ -150,7 +151,7 @@ def RunSteps(api): 'makedirs %s' % api.path.basename(skps_dir), skps_dir) # If a blacklist file exists then specify SKPs to be blacklisted. - blacklists_dir = api.vars.skia_dir.join('infra', 'bots', 'ct', 'blacklists') + blacklists_dir = infrabots_dir.join('ct', 'blacklists') blacklist_file = blacklists_dir.join( '%s_%s_%s.json' % (skia_tool, ct_page_type, skps_chromium_build)) blacklist_skps = [] @@ -173,7 +174,7 @@ def RunSteps(api): extra_variables = { 'SLAVE_NUM': str(slave_num), 'TOOL_NAME': skia_tool, - 'GIT_HASH': api.vars.got_revision, + 'GIT_HASH': got_revision, 'CONFIGURATION': api.vars.configuration, 'BUILDER': buildername, 'CHROMIUM_BUILD': skps_chromium_build, diff --git a/infra/bots/recipes/housekeeper.py b/infra/bots/recipes/housekeeper.py index 061898b5e5..3cc76c26e0 100644 --- a/infra/bots/recipes/housekeeper.py +++ b/infra/bots/recipes/housekeeper.py @@ -28,15 +28,14 @@ DEPS = [ def RunSteps(api): # Checkout, compile, etc. api.vars.setup() - api.core.checkout_bot_update() + checkout_root = api.core.default_checkout_root + got_revision = api.core.checkout_bot_update(checkout_root=checkout_root) api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) api.flavor.setup() - cwd = api.path['checkout'] - # TODO(borenet): Detect static initializers? - with api.context(cwd=cwd): + with api.context(cwd=checkout_root.join('skia')): if not api.vars.is_trybot: api.run( api.step, @@ -46,12 +45,12 @@ def RunSteps(api): now = api.time.utcnow() ts = int(calendar.timegm(now.utctimetuple())) - filename = 'nanobench_%s_%d.json' % (api.vars.got_revision, ts) - dest_dir = api.vars.perf_data_dir + filename = 'nanobench_%s_%d.json' % (got_revision, ts) + dest_dir = api.flavor.host_dirs.perf_data_dir dest_file = dest_dir + '/' + filename api.file.ensure_directory('makedirs perf_dir', dest_dir) cmd = ['python', api.core.resource('run_binary_size_analysis.py'), - '--library', api.vars.skia_out.join('Release', 'libskia.so'), + '--library', api.vars.skia_out.join('libskia.so'), '--githash', api.properties['revision'], '--dest', dest_file] if api.vars.is_trybot: diff --git a/infra/bots/recipes/infra.py b/infra/bots/recipes/infra.py index d025883192..902bf98833 100644 --- a/infra/bots/recipes/infra.py +++ b/infra/bots/recipes/infra.py @@ -20,14 +20,15 @@ DEPS = [ def RunSteps(api): api.vars.setup() - api.core.checkout_bot_update() + checkout_root = api.core.default_checkout_root + api.core.checkout_bot_update(checkout_root=checkout_root) api.infra.update_go_deps() # Run the infra tests. repo_name = api.properties['repository'].split('/')[-1] if repo_name.endswith('.git'): repo_name = repo_name[:-len('.git')] - with api.context(cwd=api.vars.checkout_root.join(repo_name), + with api.context(cwd=checkout_root.join(repo_name), env=api.infra.go_env): api.step('infra_tests', cmd=['make', '-C', 'infra/bots', 'test']) diff --git a/infra/bots/recipes/perf.py b/infra/bots/recipes/perf.py index 8179b1e83d..06d53681cd 100644 --- a/infra/bots/recipes/perf.py +++ b/infra/bots/recipes/perf.py @@ -27,6 +27,23 @@ DEPS = [ ] +def upload_perf_results(buildername): + if 'Release' not in buildername: + return False + skip_upload_bots = [ + 'ASAN', + 'Coverage', + 'MSAN', + 'TSAN', + 'UBSAN', + 'Valgrind', + ] + for s in skip_upload_bots: + if s in buildername: + return False + return True + + def nanobench_flags(api, bot): args = ['--pre_log'] @@ -240,14 +257,15 @@ def nanobench_flags(api, bot): def perf_steps(api): """Run Skia benchmarks.""" - if api.vars.upload_perf_results: + b = api.properties['buildername'] + if upload_perf_results(b): api.flavor.create_clean_device_dir( api.flavor.device_dirs.perf_data_dir) # Run nanobench. properties = [ '--properties', - 'gitHash', api.vars.got_revision, + 'gitHash', api.properties['revision'], ] if api.vars.is_trybot: properties.extend([ @@ -297,12 +315,12 @@ def perf_steps(api): '~shapes_rrect_inner_rrect_50_500x500', # skia:7551 ]) - if api.vars.upload_perf_results: + if upload_perf_results(b): now = api.time.utcnow() ts = int(calendar.timegm(now.utctimetuple())) json_path = api.flavor.device_path_join( api.flavor.device_dirs.perf_data_dir, - 'nanobench_%s_%d.json' % (api.vars.got_revision, ts)) + 'nanobench_%s_%d.json' % (api.properties['revision'], ts)) args.extend(['--outResultsFile', json_path]) args.extend(properties) @@ -320,12 +338,13 @@ def perf_steps(api): abort_on_failure=False) # Copy results to swarming out dir. - if api.vars.upload_perf_results: - api.file.ensure_directory('makedirs perf_dir', - api.path.dirname(api.vars.perf_data_dir)) + if upload_perf_results(b): + api.file.ensure_directory( + 'makedirs perf_dir', + api.path.dirname(api.flavor.host_dirs.perf_data_dir)) api.flavor.copy_directory_contents_to_host( api.flavor.device_dirs.perf_data_dir, - api.vars.perf_data_dir) + api.flavor.host_dirs.perf_data_dir) def RunSteps(api): diff --git a/infra/bots/recipes/recreate_skps.py b/infra/bots/recipes/recreate_skps.py index f20c24d977..b41d56f3f6 100644 --- a/infra/bots/recipes/recreate_skps.py +++ b/infra/bots/recipes/recreate_skps.py @@ -36,11 +36,13 @@ TEST_BUILDERS = { def RunSteps(api): # Check out Chrome. api.vars.setup() - api.core.checkout_bot_update() + checkout_root = api.core.default_checkout_root + api.core.checkout_bot_update(checkout_root=checkout_root) api.file.ensure_directory('makedirs tmp_dir', api.vars.tmp_dir) api.flavor.setup() - src_dir = api.vars.checkout_root.join('src') + src_dir = checkout_root.join('src') + skia_dir = checkout_root.join('skia') out_dir = src_dir.join('out', 'Release') with api.context(cwd=src_dir): @@ -62,7 +64,7 @@ def RunSteps(api): api.file.ensure_directory('makedirs skp_output', output_dir) # Capture the SKPs. - asset_dir = api.vars.infrabots_dir.join('assets', 'skp') + asset_dir = skia_dir.join('infra', 'bots', 'assets', 'skp') cmd = ['python', asset_dir.join('create.py'), '--chrome_src_path', src_dir, '--browser_executable', src_dir.join('out', 'Release', 'chrome'), @@ -70,16 +72,16 @@ def RunSteps(api): # TODO(rmistry): Uncomment the below after skbug.com/6797 is fixed. # if 'Canary' not in api.properties['buildername']: # cmd.append('--upload_to_partner_bucket') - with api.context(cwd=api.vars.skia_dir): + with api.context(cwd=skia_dir): api.run(api.step, 'Recreate SKPs', cmd=cmd) # Upload the SKPs. if 'Canary' not in api.properties['buildername']: api.infra.update_go_deps() cmd = ['python', - api.vars.skia_dir.join('infra', 'bots', 'upload_skps.py'), + skia_dir.join('infra', 'bots', 'upload_skps.py'), '--target_dir', output_dir] - with api.context(cwd=api.vars.skia_dir, env=api.infra.go_env): + with api.context(cwd=skia_dir, env=api.infra.go_env): api.run(api.step, 'Upload SKPs', cmd=cmd) diff --git a/infra/bots/recipes/skpbench.py b/infra/bots/recipes/skpbench.py index 0582cde2e7..69aca7c225 100644 --- a/infra/bots/recipes/skpbench.py +++ b/infra/bots/recipes/skpbench.py @@ -10,7 +10,6 @@ import calendar DEPS = [ - 'core', 'flavor', 'recipe_engine/context', 'recipe_engine/file', @@ -28,7 +27,7 @@ ADB_BINARY = 'adb.1.0.35' def _run(api, title, *cmd, **kwargs): - with api.context(cwd=api.vars.skia_dir): + with api.context(cwd=api.path['start_dir'].join('skia')): return api.run(api.step, title, cmd=list(cmd), **kwargs) @@ -40,10 +39,11 @@ def _adb(api, title, *cmd, **kwargs): def skpbench_steps(api): """benchmark Skia using skpbench.""" - api.file.ensure_directory('makedirs perf_dir', api.vars.perf_data_dir) + api.file.ensure_directory( + 'makedirs perf_dir', api.flavor.host_dirs.perf_data_dir) - app = api.vars.skia_out.join(api.vars.configuration, 'skpbench') - _adb(api, 'push skpbench', 'push', app, api.vars.android_bin_dir) + app = api.vars.skia_out.join('skpbench') + _adb(api, 'push skpbench', 'push', app, api.flavor.device_dirs.bin_dir) skpbench_dir = api.vars.slave_dir.join('skia', 'tools', 'skpbench') table = api.path.join(api.vars.swarming_out_dir, 'table') @@ -54,7 +54,7 @@ def skpbench_steps(api): config = 'gles' skpbench_args = [ - api.path.join(api.vars.android_bin_dir, 'skpbench'), + api.path.join(api.flavor.device_dirs.bin_dir, 'skpbench'), '--adb', '--adb_binary', ADB_BINARY, '--resultsfile', table, @@ -65,11 +65,10 @@ def skpbench_steps(api): skpbench_args += [ '--pr', 'ccpr', '--nocache', - api.path.join(api.vars.android_data_dir, 'skps/desk_*svg.skp'), - api.path.join(api.vars.android_data_dir, 'skps/desk_chalkboard.skp')] + api.path.join(api.flavor.device_dirs.skp_dir, 'desk_*svg.skp'), + api.path.join(api.flavor.device_dirs.skp_dir, 'desk_chalkboard.skp')] else: - skpbench_args += [ - api.path.join(api.vars.android_data_dir, 'skps')] + skpbench_args += [api.flavor.device_dirs.skp_dir] api.run(api.python, 'skpbench', script=skpbench_dir.join('skpbench.py'), @@ -78,7 +77,7 @@ def skpbench_steps(api): skiaperf_args = [ table, '--properties', - 'gitHash', api.vars.got_revision, + 'gitHash', api.properties['revision'], ] if api.vars.is_trybot: skiaperf_args.extend([ @@ -93,8 +92,8 @@ def skpbench_steps(api): now = api.time.utcnow() ts = int(calendar.timegm(now.utctimetuple())) json_path = api.path.join( - api.vars.perf_data_dir, - 'skpbench_%s_%d.json' % (api.vars.got_revision, ts)) + api.flavor.host_dirs.perf_data_dir, + 'skpbench_%s_%d.json' % (api.properties['revision'], ts)) skiaperf_args.extend([ '--outfile', json_path diff --git a/infra/bots/recipes/test.py b/infra/bots/recipes/test.py index 4d3a36eca2..d79d22a6e7 100644 --- a/infra/bots/recipes/test.py +++ b/infra/bots/recipes/test.py @@ -24,6 +24,21 @@ DEPS = [ ] +def upload_dm_results(buildername): + skip_upload_bots = [ + 'ASAN', + 'Coverage', + 'MSAN', + 'TSAN', + 'UBSAN', + 'Valgrind', + ] + for s in skip_upload_bots: + if s in buildername: + return False + return True + + def dm_flags(api, bot): args = [] configs = [] @@ -828,10 +843,11 @@ def key_params(api): def test_steps(api): """Run the DM test.""" + b = api.properties['buildername'] use_hash_file = False - if api.vars.upload_dm_results: - host_dm_dir = str(api.vars.dm_dir) - api.flavor.create_clean_host_dir(api.vars.test_dir) + if upload_dm_results(b): + host_dm_dir = str(api.flavor.host_dirs.dm_dir) + api.flavor.create_clean_host_dir(api.path['start_dir'].join('test')) device_dm_dir = str(api.flavor.device_dirs.dm_dir) if host_dm_dir != device_dm_dir: api.flavor.create_clean_device_dir(device_dm_dir) @@ -887,7 +903,7 @@ def test_steps(api): # Run DM. properties = [ - 'gitHash', api.vars.got_revision, + 'gitHash', api.properties['revision'], 'builder', api.vars.builder_name, 'buildbucket_build_id', api.properties.get('buildbucket_build_id', ''), ] @@ -931,7 +947,7 @@ def test_steps(api): if use_hash_file: args.extend(['--uninterestingHashesFile', hashes_file]) - if api.vars.upload_dm_results: + if upload_dm_results(b): args.extend(['--writePath', api.flavor.device_dirs.dm_dir]) args.extend(dm_flags(api, api.vars.builder_name)) @@ -946,10 +962,10 @@ def test_steps(api): api.run(api.flavor.step, 'dm', cmd=args, abort_on_failure=False) - if api.vars.upload_dm_results: + if upload_dm_results(b): # Copy images and JSON to host machine if needed. api.flavor.copy_directory_contents_to_host( - api.flavor.device_dirs.dm_dir, api.vars.dm_dir) + api.flavor.device_dirs.dm_dir, api.flavor.host_dirs.dm_dir) def RunSteps(api): diff --git a/infra/bots/recipes/upload_calmbench_results.py b/infra/bots/recipes/upload_calmbench_results.py index 1075703791..8aa6c01644 100644 --- a/infra/bots/recipes/upload_calmbench_results.py +++ b/infra/bots/recipes/upload_calmbench_results.py @@ -27,7 +27,7 @@ def FindFile(api, suffix): with api.context(cwd=api.path['start_dir']): results = api.file.glob_paths( 'find %s results' % suffix, - api.vars.perf_dir, + api.path['start_dir'].join('perf'), '*.%s' % suffix, test_data=['bench_modified_master.%s' % suffix]) if len(results) != 1: # pragma: nocover @@ -48,7 +48,7 @@ def RunSteps(api): csv_src = FindFile(api, "csv") ts = int(calendar.timegm(now.utctimetuple())) - basename = "bench_modified_master_%s_%d" % (api.vars.got_revision, ts) + basename = "bench_modified_master_%s_%d" % (api.properties['revision'], ts) gs_path = '/'.join(( 'calmbench-v1', str(now.year).zfill(4), |