diff options
-rw-r--r-- | infra/bots/assets/win_toolchain/VERSION | 1 | ||||
-rwxr-xr-x | infra/bots/assets/win_toolchain/common.py | 26 | ||||
-rwxr-xr-x | infra/bots/assets/win_toolchain/create.py | 128 | ||||
-rwxr-xr-x | infra/bots/assets/win_toolchain/create_and_upload.py | 49 | ||||
-rwxr-xr-x | infra/bots/assets/win_toolchain/download.py | 16 | ||||
-rwxr-xr-x | infra/bots/assets/win_toolchain/upload.py | 16 | ||||
-rw-r--r-- | infra/bots/isolate_win_toolchain.py | 124 | ||||
-rw-r--r-- | infra/bots/win_toolchain.isolate | 13 | ||||
-rw-r--r-- | infra/bots/win_toolchain_hash.json | 4 |
9 files changed, 236 insertions, 141 deletions
diff --git a/infra/bots/assets/win_toolchain/VERSION b/infra/bots/assets/win_toolchain/VERSION new file mode 100644 index 0000000000..d00491fd7e --- /dev/null +++ b/infra/bots/assets/win_toolchain/VERSION @@ -0,0 +1 @@ +1 diff --git a/infra/bots/assets/win_toolchain/common.py b/infra/bots/assets/win_toolchain/common.py new file mode 100755 index 0000000000..4920c9b4fb --- /dev/null +++ b/infra/bots/assets/win_toolchain/common.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# +# Copyright 2016 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Common vars used by scripts in this directory.""" + + +import os +import sys + +FILE_DIR = os.path.dirname(os.path.abspath(__file__)) +INFRA_BOTS_DIR = os.path.realpath(os.path.join(FILE_DIR, os.pardir, os.pardir)) + +sys.path.insert(0, INFRA_BOTS_DIR) +from assets import assets + +ASSET_NAME = os.path.basename(FILE_DIR) + + +def run(cmd): + """Run a command, eg. "upload" or "download". """ + assets.main([cmd, ASSET_NAME] + sys.argv[1:]) diff --git a/infra/bots/assets/win_toolchain/create.py b/infra/bots/assets/win_toolchain/create.py new file mode 100755 index 0000000000..4ec20d4e86 --- /dev/null +++ b/infra/bots/assets/win_toolchain/create.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# +# Copyright 2016 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Download an updated VS toolchain""" + + +import argparse +import common +import json +import os +import shlex +import shutil +import subprocess +import sys +import utils + +import win_toolchain_utils + + +# By default the toolchain includes a bunch of unnecessary stuff with long path +# names. Trim out directories with these names. +IGNORE_LIST = [ + 'WindowsMobile', + 'App Certification Kit', + 'Debuggers', + 'Extension SDKs', + 'winrt', + 'DesignTime', + 'AccChecker', +] + +REPO_CHROME = 'https://chromium.googlesource.com/chromium/src.git' + + +def filter_toolchain_files(dirname, files): + """Callback for shutil.copytree. Return lists of files to skip.""" + split = dirname.split(os.path.sep) + for ign in IGNORE_LIST: + if ign in split: + print 'Ignoring dir %s' % dirname + return files + return [] + + +def get_toolchain_dir(toolchain_dir_output): + """Find the toolchain directory.""" + prefix = 'vs_path = ' + for line in toolchain_dir_output.splitlines(): + if line.startswith(prefix): + return line[len(prefix):].strip('"') + raise Exception('Unable to find toolchain dir in output:\n%s' % ( + toolchain_dir_output)) + + +def gen_toolchain(chrome_path, msvs_version, target_dir): + """Update the VS toolchain and copy it to the target_dir.""" + with utils.chdir(os.path.join(chrome_path, 'src')): + subprocess.check_call([utils.GCLIENT, 'sync']) + depot_tools = subprocess.check_output([ + 'python', os.path.join('build', 'find_depot_tools.py')]).rstrip() + with utils.git_branch(): + vs_toolchain_py = os.path.join('build', 'vs_toolchain.py') + env = os.environ.copy() + env['GYP_MSVS_VERSION'] = msvs_version + subprocess.check_call(['python', vs_toolchain_py, 'update'], env=env) + output = subprocess.check_output(['python', vs_toolchain_py, + 'get_toolchain_dir'], env=env).rstrip() + src_dir = get_toolchain_dir(output) + # Mock out absolute paths in win_toolchain.json. + win_toolchain_utils.abstract(os.path.join('build', 'win_toolchain.json'), + os.path.dirname(depot_tools)) + + # Copy the toolchain files to the target_dir. + build = os.path.join(os.getcwd(), 'build') + dst_build = os.path.join(target_dir, 'src', 'build') + os.makedirs(dst_build) + for f in ('find_depot_tools.py', 'vs_toolchain.py', 'win_toolchain.json'): + shutil.copyfile(os.path.join(build, f), os.path.join(dst_build, f)) + + shutil.copytree(os.path.join(os.getcwd(), 'tools', 'gyp', 'pylib'), + os.path.join(target_dir, 'src', 'tools', 'gyp', 'pylib')) + + dst_depot_tools = os.path.join(target_dir, 'depot_tools') + os.makedirs(dst_depot_tools) + for f in ('gclient.py', 'breakpad.py'): + shutil.copyfile(os.path.join(depot_tools, f), + os.path.join(dst_depot_tools, f)) + toolchain_dst = os.path.join( + target_dir, 'depot_tools', os.path.relpath(src_dir, depot_tools)) + shutil.copytree(src_dir, toolchain_dst, ignore=filter_toolchain_files) + + +def create_asset(target_dir, msvs_version, chrome_path=None): + """Create the asset.""" + if not os.path.isdir(target_dir): + os.makedirs(target_dir) + with utils.tmp_dir() as tmp_dir: + if not chrome_path: + print ('Syncing Chrome from scratch. If you already have a checkout, ' + 'specify --chrome_path to save time.') + chrome_path = os.path.join(tmp_dir.name, 'src') + if not os.path.isdir(chrome_path): + subprocess.check_call([utils.GCLIENT, 'config', REPO_CHROME, '--managed']) + subprocess.check_call([utils.GCLIENT, 'sync']) + + gen_toolchain(chrome_path, msvs_version, target_dir) + +def main(): + if sys.platform != 'win32': + print >> sys.stderr, 'This script only runs on Windows.' + sys.exit(1) + + parser = argparse.ArgumentParser() + parser.add_argument('--msvs_version', required=True) + parser.add_argument('--chrome_path') + parser.add_argument('--target_dir', '-t', required=True) + args = parser.parse_args() + target_dir = os.path.abspath(args.target_dir) + create_asset(target_dir, args.msvs_version, args.chrome_path) + + +if __name__ == '__main__': + main() diff --git a/infra/bots/assets/win_toolchain/create_and_upload.py b/infra/bots/assets/win_toolchain/create_and_upload.py new file mode 100755 index 0000000000..d428ca4ef9 --- /dev/null +++ b/infra/bots/assets/win_toolchain/create_and_upload.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# +# Copyright 2016 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Create the asset and upload it.""" + + +import argparse +import common +import os +import subprocess +import sys +import utils + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--gsutil') + parser.add_argument('--chrome_path') + parser.add_argument('--msvs_version', required=True) + args = parser.parse_args() + + with utils.tmp_dir(): + cwd = os.getcwd() + create_script = os.path.join(common.FILE_DIR, 'create.py') + upload_script = os.path.join(common.FILE_DIR, 'upload.py') + + try: + cmd = ['python', create_script, + '-t', cwd, + '--msvs_version', args.msvs_version] + if args.chrome_path: + cmd.extend(['--chrome_path', args.chrome_path]) + subprocess.check_call(cmd) + cmd = ['python', upload_script, '-t', cwd] + if args.gsutil: + cmd.extend(['--gsutil', args.gsutil]) + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + # Trap exceptions to avoid printing two stacktraces. + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/infra/bots/assets/win_toolchain/download.py b/infra/bots/assets/win_toolchain/download.py new file mode 100755 index 0000000000..96cc87d43f --- /dev/null +++ b/infra/bots/assets/win_toolchain/download.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# +# Copyright 2016 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Download the current version of the asset.""" + + +import common + + +if __name__ == '__main__': + common.run('download') diff --git a/infra/bots/assets/win_toolchain/upload.py b/infra/bots/assets/win_toolchain/upload.py new file mode 100755 index 0000000000..ba7fc8b6a1 --- /dev/null +++ b/infra/bots/assets/win_toolchain/upload.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# +# Copyright 2016 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Upload a new version of the asset.""" + + +import common + + +if __name__ == '__main__': + common.run('upload') diff --git a/infra/bots/isolate_win_toolchain.py b/infra/bots/isolate_win_toolchain.py deleted file mode 100644 index 696077992f..0000000000 --- a/infra/bots/isolate_win_toolchain.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - - -"""Download an updated VS toolchain, isolate it, upload a CL to update Skia.""" - - -import argparse -import json -import os -import shlex -import shutil -import subprocess -import sys -import utils - -import win_toolchain_utils - - -REPO_CHROME = 'https://chromium.googlesource.com/chromium/src.git' -REPO_SKIA = 'https://skia.googlesource.com/skia.git' - - -def get_toolchain_dir(toolchain_dir_output): - """Find the toolchain directory.""" - prefix = 'vs_path = ' - for line in toolchain_dir_output.splitlines(): - if line.startswith(prefix): - return line[len(prefix):].strip('"') - raise Exception('Unable to find toolchain dir in output:\n%s' % ( - toolchain_dir_output)) - - -def gen_toolchain(chrome_path, msvs_version, isolate_file): - """Update the VS toolchain, isolate it, and return the isolated hash.""" - with utils.chdir(chrome_path): - subprocess.check_call([utils.GCLIENT, 'sync']) - depot_tools = subprocess.check_output([ - 'python', os.path.join('build', 'find_depot_tools.py')]).rstrip() - with utils.git_branch(): - vs_toolchain_py = os.path.join('build', 'vs_toolchain.py') - env = os.environ.copy() - env['GYP_MSVS_VERSION'] = msvs_version - subprocess.check_call(['python', vs_toolchain_py, 'update'], env=env) - output = subprocess.check_output(['python', vs_toolchain_py, - 'get_toolchain_dir'], env=env).rstrip() - src_dir = get_toolchain_dir(output) - # Mock out absolute paths in win_toolchain.json. - win_toolchain_utils.abstract(os.path.join('build', 'win_toolchain.json'), - os.path.dirname(depot_tools)) - - # Isolate the toolchain. Assumes we're running on Windows, since the above - # would fail otherwise. - isolate_file_dirname = os.path.dirname(isolate_file) - toolchain_relpath = os.path.relpath(src_dir, isolate_file_dirname) - chrome_relpath = os.path.relpath(os.getcwd(), isolate_file_dirname) - depot_tools_relpath = os.path.relpath(depot_tools, isolate_file_dirname) - isolate = os.path.join( - os.curdir, 'tools', 'luci-go', 'win64', 'isolate.exe') - isolate_cmd = [isolate, 'archive', '--quiet', - '--isolate-server', 'https://isolateserver.appspot.com', - '-i', isolate_file, - '-s', 'win_toolchain_%s.isolated' % msvs_version, - '--extra-variable', 'WIN_TOOLCHAIN_DIR=%s' % toolchain_relpath, - '--extra-variable', 'DEPOT_TOOLS_DIR=%s' % depot_tools_relpath, - '--extra-variable', 'CHROME_DIR=%s' % chrome_relpath] - isolate_out = subprocess.check_output(isolate_cmd).rstrip() - return shlex.split(isolate_out)[0] - - -def update_toolchain_file(skia_path, msvs_version, isolated_hash): - """Edit the win_toolchain_hash file, upload a CL.""" - with utils.chdir(skia_path): - with utils.git_branch(): - hash_file = os.path.join('infra', 'bots', 'win_toolchain_hash.json') - with open(hash_file) as f: - hashes = json.load(f) - hashes[msvs_version] = isolated_hash - with open(hash_file, 'w') as f: - json.dump(hashes, f, indent=4, sort_keys=True) - subprocess.check_call([utils.GIT, 'add', hash_file]) - subprocess.check_call([utils.GIT, 'commit', '-m', 'Update Win toolchain']) - subprocess.check_call([utils.GIT, 'cl', 'upload', '--bypass-hooks']) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('--msvs_version', required=True) - parser.add_argument('--chrome_path') - parser.add_argument('--skia_path') - args = parser.parse_args() - - isolate_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), - 'win_toolchain.isolate') - - with utils.print_timings(): - with utils.tmp_dir() as tmp_dir: - chrome_path = args.chrome_path - if not chrome_path: - print ('Syncing Chrome from scratch. If you already have a checkout, ' - 'specify --chrome_path to save time.') - chrome_path = os.path.join(tmp_dir.name, 'src') - if not os.path.isdir(chrome_path): - utils.git_clone(REPO_CHROME, chrome_path) - - skia_path = args.skia_path - if not skia_path: - print ('Syncing Skia from scratch. If you already have a checkout, ' - 'specify --chrome_path to save time.') - skia_path = os.path.join(tmp_dir.name, 'skia') - if not os.path.isdir(skia_path): - utils.git_clone(REPO_SKIA, skia_path) - - isolated_hash = gen_toolchain(chrome_path, args.msvs_version, - isolate_file) - update_toolchain_file(skia_path, args.msvs_version, isolated_hash) - - -if __name__ == '__main__': - main() diff --git a/infra/bots/win_toolchain.isolate b/infra/bots/win_toolchain.isolate deleted file mode 100644 index a2c91db0bb..0000000000 --- a/infra/bots/win_toolchain.isolate +++ /dev/null @@ -1,13 +0,0 @@ -{ - 'variables': { - 'files': [ - '<(CHROME_DIR)/build/find_depot_tools.py', - '<(CHROME_DIR)/build/vs_toolchain.py', - '<(CHROME_DIR)/build/win_toolchain.json', - '<(CHROME_DIR)/tools/gyp/pylib/', - '<(DEPOT_TOOLS_DIR)/gclient.py', - '<(DEPOT_TOOLS_DIR)/breakpad.py', - '<(WIN_TOOLCHAIN_DIR)/', - ], - }, -} diff --git a/infra/bots/win_toolchain_hash.json b/infra/bots/win_toolchain_hash.json deleted file mode 100644 index fb759d6688..0000000000 --- a/infra/bots/win_toolchain_hash.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "2013": "705384d88f80da637eb367e5acc6f315c0e1db2f", - "2015": "a59d15e51c043be61a559315f2110ac15969aa86" -}
\ No newline at end of file |