aboutsummaryrefslogtreecommitdiffhomepage
path: root/infra
diff options
context:
space:
mode:
authorGravatar borenet <borenet@chromium.org>2016-03-10 07:01:39 -0800
committerGravatar Commit bot <commit-bot@chromium.org>2016-03-10 07:01:39 -0800
commit2bcfb4bec33efc0d113dc7d21830dd69f6b83ad0 (patch)
treef060dc025ce985d72e77e306bc64eb5a25f531c5 /infra
parentf1d746c188ede847968efafde89c8a5501d45c7d (diff)
Add isolate_win_toolchain.py
Intended to be run manually by a developer on Windows. Downloads the requested Windows toolchain, uploads it to the isolate server, records the isolated hash in the JSON file and uploads a CL to update that file. BUG=skia:4553, skia:4763 GOLD_TRYBOT_URL= https://gold.skia.org/search2?unt=true&query=source_type%3Dgm&master=false&issue=1775073002 Review URL: https://codereview.chromium.org/1775073002
Diffstat (limited to 'infra')
-rw-r--r--infra/bots/isolate_win_toolchain.py109
-rw-r--r--infra/bots/utils.py177
-rw-r--r--infra/bots/win_toolchain.isolate7
-rw-r--r--infra/bots/win_toolchain_hash.json4
4 files changed, 297 insertions, 0 deletions
diff --git a/infra/bots/isolate_win_toolchain.py b/infra/bots/isolate_win_toolchain.py
new file mode 100644
index 0000000000..ff01bd60d8
--- /dev/null
+++ b/infra/bots/isolate_win_toolchain.py
@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+import argparse
+import json
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+import utils
+
+
+REPO_CHROME = 'https://chromium.googlesource.com/chromium/src.git'
+REPO_SKIA = 'https://skia.googlesource.com/skia.git'
+
+
+def get_toolchain_dir(toolchain_dir_output):
+ """Find the toolchain directory."""
+ prefix = 'vs_path = '
+ for line in toolchain_dir_output.splitlines():
+ if line.startswith(prefix):
+ return line[len(prefix):].strip('"')
+ raise Exception('Unable to find toolchain dir in output:\n%s' % (
+ toolchain_dir_output))
+
+
+def gen_toolchain(chrome_path, msvs_version, isolate_file):
+ """Update the VS toolchain, isolate it, and return the isolated hash."""
+ with utils.chdir(chrome_path):
+ subprocess.check_call([utils.GCLIENT, 'sync'])
+ with utils.git_branch():
+ vs_toolchain_py = os.path.join('build', 'vs_toolchain.py')
+ env = os.environ.copy()
+ env['GYP_MSVS_VERSION'] = msvs_version
+ subprocess.check_call(['python', vs_toolchain_py, 'update'], env=env)
+ output = subprocess.check_output(['python', vs_toolchain_py,
+ 'get_toolchain_dir'], env=env).rstrip()
+ src_dir = get_toolchain_dir(output)
+
+ # Isolate the toolchain. Assumes we're running on Windows, since the above
+ # would fail otherwise.
+ rel_path = os.path.relpath(src_dir, os.path.dirname(isolate_file))
+ isolate = os.path.join(
+ os.curdir, 'tools', 'luci-go', 'win64', 'isolate.exe')
+ isolate_cmd = [isolate, 'archive', '--quiet',
+ '--isolate-server', 'https://isolateserver.appspot.com',
+ '-i', isolate_file,
+ '-s', 'win_toolchain_%s.isolated' % msvs_version,
+ '--extra-variable', 'WIN_TOOLCHAIN_DIR=%s' % rel_path]
+ isolate_out = subprocess.check_output(isolate_cmd).rstrip()
+ return shlex.split(isolate_out)[0]
+
+
+def update_toolchain_file(skia_path, msvs_version, isolated_hash):
+ """Edit the win_toolchain_hash file, upload a CL."""
+ with utils.chdir(skia_path):
+ with utils.git_branch():
+ hash_file = os.path.join('infra', 'bots', 'win_toolchain_hash.json')
+ with open(hash_file) as f:
+ hashes = json.load(f)
+ hashes[msvs_version] = isolated_hash
+ with open(hash_file, 'w') as f:
+ json.dump(hashes, f, indent=4, sort_keys=True)
+ subprocess.check_call([utils.GIT, 'add', hash_file])
+ subprocess.check_call([utils.GIT, 'commit', '-m', 'Update Win toolchain'])
+ subprocess.check_call([utils.GIT, 'cl', 'upload', '--bypass-hooks'])
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--msvs_version', required=True)
+ parser.add_argument('--chrome_path')
+ parser.add_argument('--skia_path')
+ args = parser.parse_args()
+
+ isolate_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+ 'win_toolchain.isolate')
+
+ with utils.print_timings():
+ with utils.tmp_dir() as tmp_dir:
+ chrome_path = args.chrome_path
+ if not chrome_path:
+ print ('Syncing Chrome from scratch. If you already have a checkout, '
+ 'specify --chrome_path to save time.')
+ chrome_path = os.path.join(tmp_dir.name, 'src')
+ if not os.path.isdir(chrome_path):
+ utils.git_clone(REPO_CHROME, chrome_path)
+
+ skia_path = args.skia_path
+ if not skia_path:
+ print ('Syncing Skia from scratch. If you already have a checkout, '
+ 'specify --chrome_path to save time.')
+ skia_path = os.path.join(tmp_dir.name, 'skia')
+ if not os.path.isdir(skia_path):
+ utils.git_clone(REPO_SKIA, skia_path)
+
+ isolated_hash = gen_toolchain(chrome_path, args.msvs_version,
+ isolate_file)
+ update_toolchain_file(skia_path, args.msvs_version, isolated_hash)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/infra/bots/utils.py b/infra/bots/utils.py
index ee648a26a4..a60ebe97b8 100644
--- a/infra/bots/utils.py
+++ b/infra/bots/utils.py
@@ -7,6 +7,18 @@
import datetime
+import errno
+import os
+import shutil
+import sys
+import subprocess
+import tempfile
+import time
+import uuid
+
+
+GCLIENT = 'gclient.bat' if sys.platform == 'win32' else 'gclient'
+GIT = 'git.bat' if sys.platform == 'win32' else 'git'
class print_timings(object):
@@ -21,3 +33,168 @@ class print_timings(object):
finish = datetime.datetime.utcnow()
duration = (finish-self._start).total_seconds()
print 'Task finished at %s GMT (%f seconds)' % (str(finish), duration)
+
+
+class tmp_dir(object):
+ """Helper class used for creating a temporary directory and working in it."""
+ def __init__(self):
+ self._orig_dir = None
+ self._tmp_dir = None
+
+ def __enter__(self):
+ self._orig_dir = os.getcwd()
+ self._tmp_dir = tempfile.mkdtemp()
+ os.chdir(self._tmp_dir)
+ return self
+
+ def __exit__(self, t, v, tb):
+ os.chdir(self._orig_dir)
+ RemoveDirectory(self._tmp_dir)
+
+ @property
+ def name(self):
+ return self._tmp_dir
+
+
+class chdir(object):
+ """Helper class used for changing into and out of a directory."""
+ def __init__(self, d):
+ self._dir = d
+ self._orig_dir = None
+
+ def __enter__(self):
+ self._orig_dir = os.getcwd()
+ os.chdir(self._dir)
+ return self
+
+ def __exit__(self, t, v, tb):
+ os.chdir(self._orig_dir)
+
+
+def git_clone(repo_url, dest_dir):
+ """Clone the given repo into the given destination directory."""
+ subprocess.check_call([GIT, 'clone', repo_url, dest_dir])
+
+
+class git_branch(object):
+ """Check out a temporary git branch.
+
+ On exit, deletes the branch and attempts to restore the original state.
+ """
+ def __init__(self):
+ self._branch = None
+ self._orig_branch = None
+ self._stashed = False
+
+ def __enter__(self):
+ output = subprocess.check_output([GIT, 'stash'])
+ self._stashed = 'No local changes' not in output
+
+ # Get the original branch name or commit hash.
+ self._orig_branch = subprocess.check_output([
+ GIT, 'rev-parse', '--abbrev-ref', 'HEAD']).rstrip()
+ if self._orig_branch == 'HEAD':
+ self._orig_branch = subprocess.check_output([
+ GIT, 'rev-parse', 'HEAD']).rstrip()
+
+ # Check out a new branch, based at updated origin/master.
+ subprocess.check_call([GIT, 'fetch', 'origin'])
+ self._branch = '_tmp_%s' % uuid.uuid4()
+ subprocess.check_call([GIT, 'checkout', '-b', self._branch,
+ '-t', 'origin/master'])
+ return self
+
+ def __exit__(self, exc_type, _value, _traceback):
+ subprocess.check_call([GIT, 'reset', '--hard', 'HEAD'])
+ subprocess.check_call([GIT, 'checkout', self._orig_branch])
+ if self._stashed:
+ subprocess.check_call([GIT, 'stash', 'pop'])
+ subprocess.check_call([GIT, 'branch', '-D', self._branch])
+
+
+def RemoveDirectory(*path):
+ """Recursively removes a directory, even if it's marked read-only.
+
+ This was copied from:
+ https://chromium.googlesource.com/chromium/tools/build/+/f3e7ff03613cd59a463b2ccc49773c3813e77404/scripts/common/chromium_utils.py#491
+
+ Remove the directory located at *path, if it exists.
+
+ shutil.rmtree() doesn't work on Windows if any of the files or directories
+ are read-only, which svn repositories and some .svn files are. We need to
+ be able to force the files to be writable (i.e., deletable) as we traverse
+ the tree.
+
+ Even with all this, Windows still sometimes fails to delete a file, citing
+ a permission error (maybe something to do with antivirus scans or disk
+ indexing). The best suggestion any of the user forums had was to wait a
+ bit and try again, so we do that too. It's hand-waving, but sometimes it
+ works. :/
+ """
+ file_path = os.path.join(*path)
+ if not os.path.exists(file_path):
+ return
+
+ if sys.platform == 'win32':
+ # Give up and use cmd.exe's rd command.
+ file_path = os.path.normcase(file_path)
+ for _ in xrange(3):
+ print 'RemoveDirectory running %s' % (' '.join(
+ ['cmd.exe', '/c', 'rd', '/q', '/s', file_path]))
+ if not subprocess.call(['cmd.exe', '/c', 'rd', '/q', '/s', file_path]):
+ break
+ print ' Failed'
+ time.sleep(3)
+ return
+
+ def RemoveWithRetry_non_win(rmfunc, path):
+ if os.path.islink(path):
+ return os.remove(path)
+ else:
+ return rmfunc(path)
+
+ remove_with_retry = RemoveWithRetry_non_win
+
+ def RmTreeOnError(function, path, excinfo):
+ r"""This works around a problem whereby python 2.x on Windows has no ability
+ to check for symbolic links. os.path.islink always returns False. But
+ shutil.rmtree will fail if invoked on a symbolic link whose target was
+ deleted before the link. E.g., reproduce like this:
+ > mkdir test
+ > mkdir test\1
+ > mklink /D test\current test\1
+ > python -c "import chromium_utils; chromium_utils.RemoveDirectory('test')"
+ To avoid this issue, we pass this error-handling function to rmtree. If
+ we see the exact sort of failure, we ignore it. All other failures we re-
+ raise.
+ """
+
+ exception_type = excinfo[0]
+ exception_value = excinfo[1]
+ # If shutil.rmtree encounters a symbolic link on Windows, os.listdir will
+ # fail with a WindowsError exception with an ENOENT errno (i.e., file not
+ # found). We'll ignore that error. Note that WindowsError is not defined
+ # for non-Windows platforms, so we use OSError (of which it is a subclass)
+ # to avoid lint complaints about an undefined global on non-Windows
+ # platforms.
+ if (function is os.listdir) and issubclass(exception_type, OSError):
+ if exception_value.errno == errno.ENOENT:
+ # File does not exist, and we're trying to delete, so we can ignore the
+ # failure.
+ print 'WARNING: Failed to list %s during rmtree. Ignoring.\n' % path
+ else:
+ raise
+ else:
+ raise
+
+ for root, dirs, files in os.walk(file_path, topdown=False):
+ # For POSIX: making the directory writable guarantees removability.
+ # Windows will ignore the non-read-only bits in the chmod value.
+ os.chmod(root, 0770)
+ for name in files:
+ remove_with_retry(os.remove, os.path.join(root, name))
+ for name in dirs:
+ remove_with_retry(lambda p: shutil.rmtree(p, onerror=RmTreeOnError),
+ os.path.join(root, name))
+
+ remove_with_retry(os.rmdir, file_path)
diff --git a/infra/bots/win_toolchain.isolate b/infra/bots/win_toolchain.isolate
new file mode 100644
index 0000000000..d9505bab27
--- /dev/null
+++ b/infra/bots/win_toolchain.isolate
@@ -0,0 +1,7 @@
+{
+ 'variables': {
+ 'files': [
+ '<(WIN_TOOLCHAIN_DIR)/',
+ ],
+ },
+}
diff --git a/infra/bots/win_toolchain_hash.json b/infra/bots/win_toolchain_hash.json
new file mode 100644
index 0000000000..337fd4fe6e
--- /dev/null
+++ b/infra/bots/win_toolchain_hash.json
@@ -0,0 +1,4 @@
+{
+ "2013": "d6b889963f8a6896d03457c52392f7f97d6cb94c",
+ "2015": "081ab2320c1f76e234696dd2fc8aab44fa569a8a"
+} \ No newline at end of file