From 3f39bf8fe596f87120d189dbec2889b644f07e2d Mon Sep 17 00:00:00 2001 From: Ben Wagner Date: Fri, 8 Dec 2017 13:56:54 -0500 Subject: Add Goma support in gn_flavor. Using full paths in skia.h causes "file not found" with Goma. All other Builds seem fine without the path, so I changed find_headers.py to use the basename. Change-Id: Ib520e91a92ebffe36a736eb53f643d359f5bb2ce Reviewed-on: https://skia-review.googlesource.com/79360 Reviewed-by: Greg Daniel Reviewed-by: Eric Boren Commit-Queue: Ben Wagner --- infra/bots/jobs.json | 1 + infra/bots/recipe_modules/flavor/__init__.py | 1 + .../Build-Win-Clang-x86_64-Release-Goma.json | 196 +++++++++++++++++++++ infra/bots/recipe_modules/flavor/examples/full.py | 1 + infra/bots/recipe_modules/flavor/gn_flavor.py | 87 +++++++-- infra/bots/tasks.json | 39 ++++ 6 files changed, 310 insertions(+), 15 deletions(-) create mode 100644 infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Release-Goma.json (limited to 'infra/bots') diff --git a/infra/bots/jobs.json b/infra/bots/jobs.json index e796d5b840..a362715a30 100644 --- a/infra/bots/jobs.json +++ b/infra/bots/jobs.json @@ -65,6 +65,7 @@ "Build-Win-Clang-arm64-Release-Android", "Build-Win-Clang-x86_64-Debug", "Build-Win-Clang-x86_64-Debug-ANGLE", + "Build-Win-Clang-x86_64-Debug-Goma", "Build-Win-Clang-x86_64-Debug-Vulkan", "Build-Win-Clang-x86_64-Release", "Build-Win-Clang-x86_64-Release-ANGLE", diff --git a/infra/bots/recipe_modules/flavor/__init__.py b/infra/bots/recipe_modules/flavor/__init__.py index de628501bf..aade7cec2c 100644 --- a/infra/bots/recipe_modules/flavor/__init__.py +++ b/infra/bots/recipe_modules/flavor/__init__.py @@ -5,6 +5,7 @@ DEPS = [ 'builder_name_schema', 'depot_tools/bot_update', + 'depot_tools/cipd', 'env', 'recipe_engine/context', 'recipe_engine/file', diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Release-Goma.json b/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Release-Goma.json new file mode 100644 index 0000000000..cfad3a5758 --- /dev/null +++ b/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Release-Goma.json @@ -0,0 +1,196 @@ +[ + { + "cmd": [ + "python", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[START_DIR]/tmp" + ], + "infra_step": true, + "name": "makedirs tmp_dir" + }, + { + "cmd": [ + "python", + "-u", + "\nimport os\nimport sys\nimport urllib2\n\nTOKEN_URL = (\n 'http://metadata/computeMetadata/v1/project/attributes/jwt_service_account_goma-client')\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nwith open(sys.argv[1], 'w') as f:\n f.write(contents)\n", + "[START_DIR]/tmp/jwt_service_account_goma-client.json" + ], + "infra_step": true, + "name": "download jwt_service_account_goma-client.json", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@import os@@@", + "@@@STEP_LOG_LINE@python.inline@import sys@@@", + "@@@STEP_LOG_LINE@python.inline@import urllib2@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = (@@@", + "@@@STEP_LOG_LINE@python.inline@ 'http://metadata/computeMetadata/v1/project/attributes/jwt_service_account_goma-client')@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@", + "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@", + "@@@STEP_LOG_LINE@python.inline@@@@", + "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@", + "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@", + "@@@STEP_LOG_END@python.inline@@@" + ] + }, + { + "cmd": [ + "cipd", + "ensure", + "-root", + "[CACHE]/goma", + "-ensure-file", + "infra_internal/goma/client/linux-amd64 release", + "-json-output", + "/path/to/tmp/json", + "-service-account-json", + "[START_DIR]/tmp/jwt_service_account_goma-client.json" + ], + "name": "ensure_installed", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-release---------\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra_internal/goma/client/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "python", + "-u", + "goma_ctl.py", + "ensure_start" + ], + "cwd": "[CACHE]/goma", + "env": { + "BUILDTYPE": "Release_x64", + "CHROME_HEADLESS": "1", + "GOMA_FALLBACK": "0", + "GOMA_HERMETIC": "error", + "GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json", + "GOMA_USE_LOCAL": "0", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma" + }, + "infra_step": true, + "name": "start goma" + }, + { + "cmd": [ + "python", + "-u", + "[CUSTOM_C:\\_B_WORK]/skia/bin/fetch-gn" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]/skia", + "env": { + "BUILDTYPE": "Release_x64", + "CHROME_HEADLESS": "1", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma" + }, + "infra_step": true, + "name": "fetch-gn" + }, + { + "cmd": [ + "[CUSTOM_C:\\_B_WORK]/skia/bin/gn.exe", + "gen", + "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma/Release_x64", + "--args=cc=\"clang\" cc_wrapper=\"[CACHE]/goma/gomacc\" clang_win=\"[START_DIR]/clang_win\" cxx=\"clang++\" is_debug=false target_cpu=\"x86_64\" win_sdk=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/a9e1098bba66d2acccc377d5ee81265910f29272/win_sdk\" win_vc=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/a9e1098bba66d2acccc377d5ee81265910f29272/VC\"" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]/skia", + "env": { + "BUILDTYPE": "Release_x64", + "CHROME_HEADLESS": "1", + "GOMA_FALLBACK": "0", + "GOMA_HERMETIC": "error", + "GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json", + "GOMA_USE_LOCAL": "0", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma" + }, + "name": "gn gen" + }, + { + "cmd": [ + "ninja.exe", + "-k", + "0", + "-C", + "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma/Release_x64", + "-j", + "100" + ], + "cwd": "[CUSTOM_C:\\_B_WORK]/skia", + "env": { + "BUILDTYPE": "Release_x64", + "CHROME_HEADLESS": "1", + "GOMA_FALLBACK": "0", + "GOMA_HERMETIC": "error", + "GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json", + "GOMA_USE_LOCAL": "0", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma" + }, + "name": "ninja" + }, + { + "cmd": [ + "python", + "-u", + "goma_ctl.py", + "stat" + ], + "cwd": "[CACHE]/goma", + "env": { + "BUILDTYPE": "Release_x64", + "CHROME_HEADLESS": "1", + "GOMA_FALLBACK": "0", + "GOMA_HERMETIC": "error", + "GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json", + "GOMA_USE_LOCAL": "0", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma" + }, + "infra_step": true, + "name": "print goma stats" + }, + { + "cmd": [ + "python", + "-u", + "goma_ctl.py", + "stop" + ], + "cwd": "[CACHE]/goma", + "env": { + "BUILDTYPE": "Release_x64", + "CHROME_HEADLESS": "1", + "GOMA_FALLBACK": "0", + "GOMA_HERMETIC": "error", + "GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json", + "GOMA_USE_LOCAL": "0", + "PATH": ":RECIPE_PACKAGE_REPO[depot_tools]", + "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma" + }, + "infra_step": true, + "name": "stop goma" + }, + { + "name": "$result", + "recipe_result": null, + "status_code": 0 + } +] \ No newline at end of file diff --git a/infra/bots/recipe_modules/flavor/examples/full.py b/infra/bots/recipe_modules/flavor/examples/full.py index 6a052ed058..4ec7e8fef0 100644 --- a/infra/bots/recipe_modules/flavor/examples/full.py +++ b/infra/bots/recipe_modules/flavor/examples/full.py @@ -77,6 +77,7 @@ TEST_BUILDERS = [ 'Build-Win-Clang-arm64-Release-Android', 'Build-Win-Clang-x86_64-Debug-GDI', 'Build-Win-Clang-x86_64-Release', + 'Build-Win-Clang-x86_64-Release-Goma', 'Build-Win-Clang-x86_64-Release-Vulkan', 'Build-Win-MSVC-x86-Debug-Exceptions', 'Housekeeper-PerCommit-CheckGeneratedFiles', diff --git a/infra/bots/recipe_modules/flavor/gn_flavor.py b/infra/bots/recipe_modules/flavor/gn_flavor.py index 62eb15ffa5..9e8e22baae 100644 --- a/infra/bots/recipe_modules/flavor/gn_flavor.py +++ b/infra/bots/recipe_modules/flavor/gn_flavor.py @@ -22,6 +22,36 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): '--output-dir', self.m.vars.skia_out.join(self.m.vars.configuration), '--no-sync', '--make-output-dir']) + def _get_goma_json(self): + json_key = 'jwt_service_account_goma-client' + json_filename = json_key + '.json' + + # Ensure that the tmp_dir exists. + self.m.run.run_once(self.m.file.ensure_directory, + 'makedirs tmp_dir', + self.m.vars.tmp_dir) + + json_file = self.m.vars.tmp_dir.join(json_filename) + self.m.python.inline( + 'download ' + json_filename, + """ +import os +import sys +import urllib2 + +TOKEN_URL = ( + 'http://metadata/computeMetadata/v1/project/attributes/%s') + +req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'}) +contents = urllib2.urlopen(req).read() + +with open(sys.argv[1], 'w') as f: + f.write(contents) +""" % json_key, + args=[json_file], + infra_step=True) + return json_file + def compile(self, unused_target): """Build Skia with GN.""" compiler = self.m.vars.builder_cfg.get('compiler', '') @@ -30,6 +60,7 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): os = self.m.vars.builder_cfg.get('os', '') target_arch = self.m.vars.builder_cfg.get('target_arch', '') + goma_dir = None clang_linux = str(self.m.vars.slave_dir.join('clang_linux')) emscripten_sdk = str(self.m.vars.slave_dir.join('emscripten_sdk')) linux_vulkan_sdk = str(self.m.vars.slave_dir.join('linux_vulkan_sdk')) @@ -81,6 +112,8 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): extra_ldflags.append('-L' + clang_linux + '/msan') args = {} + ninja_args = ['-k', '0', '-C', self.out_dir] + env = {} if configuration != 'Debug': args['is_debug'] = 'false' @@ -130,6 +163,21 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): 'skia_use_icu': 'false', 'skia_enable_gpu': 'false', }) + if 'Goma' in extra_config: + json_file = self._get_goma_json() + self.m.cipd.set_service_account_credentials(json_file) + goma_package = ('infra_internal/goma/client/%s' % + self.m.cipd.platform_suffix()) + goma_dir = self.m.path['cache'].join('goma') + self.m.cipd.ensure(goma_dir, {goma_package: 'release'}) + env['GOMA_SERVICE_ACCOUNT_JSON_FILE'] = json_file + env['GOMA_HERMETIC'] = 'error' + env['GOMA_USE_LOCAL'] = '0' + env['GOMA_FALLBACK'] = '0' + with self.m.context(cwd=goma_dir, env=env): + self._py('start goma', 'goma_ctl.py', args=['ensure_start']) + args['cc_wrapper'] = '"%s"' % goma_dir.join('gomacc') + ninja_args.extend(['-j', '100']) sanitize = '' if 'SAN' in extra_config: @@ -159,21 +207,30 @@ class GNFlavorUtils(default_flavor.DefaultFlavorUtils): ninja = 'ninja.exe' if 'Win' in os else 'ninja' gn = self.m.vars.skia_dir.join('bin', gn) - with self.m.context(cwd=self.m.vars.skia_dir): - self._py('fetch-gn', self.m.vars.skia_dir.join('bin', 'fetch-gn')) - env = {} - if 'CheckGeneratedFiles' in extra_config: - env['PATH'] = '%s:%%(PATH)s' % self.m.vars.skia_dir.join('bin') - self._py( - 'fetch-clang-format', - self.m.vars.skia_dir.join('bin', 'fetch-clang-format')) - if target_arch == 'wasm': - fastcomp = emscripten_sdk + '/clang/fastcomp/build_incoming_64/bin' - env['PATH'] = '%s:%%(PATH)s' % fastcomp - - with self.m.env(env): - self._run('gn gen', [gn, 'gen', self.out_dir, '--args=' + gn_args]) - self._run('ninja', [ninja, '-k', '0', '-C', self.out_dir]) + try: + with self.m.context(cwd=self.m.vars.skia_dir): + self._py('fetch-gn', self.m.vars.skia_dir.join('bin', 'fetch-gn')) + if 'CheckGeneratedFiles' in extra_config: + env['PATH'] = '%s:%%(PATH)s' % self.m.vars.skia_dir.join('bin') + self._py( + 'fetch-clang-format', + self.m.vars.skia_dir.join('bin', 'fetch-clang-format')) + if target_arch == 'wasm': + fastcomp = emscripten_sdk + '/clang/fastcomp/build_incoming_64/bin' + env['PATH'] = '%s:%%(PATH)s' % fastcomp + + with self.m.env(env): + self._run('gn gen', [gn, 'gen', self.out_dir, '--args=' + gn_args]) + self._run('ninja', [ninja] + ninja_args) + finally: + if goma_dir: + with self.m.context(cwd=goma_dir, env=env): + self.m.run(self.m.python, 'print goma stats', + script='goma_ctl.py', args=['stat'], infra_step=True, + abort_on_failure=False, fail_build_on_failure=False) + self.m.run(self.m.python, 'stop goma', + script='goma_ctl.py', args=['stop'], infra_step=True, + abort_on_failure=False, fail_build_on_failure=False) def copy_extra_build_products(self, swarming_out_dir): configuration = self.m.vars.builder_cfg.get('configuration', '') diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json index 1ebc8c3366..a5dca0490a 100644 --- a/infra/bots/tasks.json +++ b/infra/bots/tasks.json @@ -401,6 +401,12 @@ "Build-Win-Clang-x86_64-Debug-ANGLE" ] }, + "Build-Win-Clang-x86_64-Debug-Goma": { + "priority": 0.8, + "tasks": [ + "Build-Win-Clang-x86_64-Debug-Goma" + ] + }, "Build-Win-Clang-x86_64-Debug-Vulkan": { "priority": 0.8, "tasks": [ @@ -4785,6 +4791,39 @@ "isolate": "compile_skia.isolate", "priority": 0.8 }, + "Build-Win-Clang-x86_64-Debug-Goma": { + "cipd_packages": [ + { + "name": "skia/bots/clang_win", + "path": "clang_win", + "version": "version:2" + } + ], + "dependencies": [ + "Housekeeper-PerCommit-IsolateWinToolchain" + ], + "dimensions": [ + "cpu:x86-64-Haswell_GCE", + "gpu:none", + "os:Windows-2016Server-14393", + "pool:Skia" + ], + "extra_args": [ + "--workdir", + "../../..", + "compile", + "repository=<(REPO)", + "buildername=Build-Win-Clang-x86_64-Debug-Goma", + "swarm_out_dir=${ISOLATED_OUTDIR}", + "revision=<(REVISION)", + "patch_repo=<(PATCH_REPO)", + "patch_storage=<(PATCH_STORAGE)", + "patch_issue=<(ISSUE)", + "patch_set=<(PATCHSET)" + ], + "isolate": "compile_skia.isolate", + "priority": 0.8 + }, "Build-Win-Clang-x86_64-Debug-Vulkan": { "cipd_packages": [ { -- cgit v1.2.3