Add isolate_win_toolchain.py

Intended to be run manually by a developer on Windows.
Downloads the requested Windows toolchain, uploads it
to the isolate server, records the isolated hash in
the JSON file and uploads a CL to update that file.

BUG=skia:4553, skia:4763
GOLD_TRYBOT_URL= https://gold.skia.org/search2?unt=true&query=source_type%3Dgm&master=false&issue=1775073002

Review URL: https://codereview.chromium.org/1775073002

4 files changed, 297 insertions, 0 deletions

diff --git a/infra/bots/isolate_win_toolchain.py b/infra/bots/isolate_win_toolchain.py
new file mode 100644
index 0000000000..ff01bd60d8
--- /dev/null
+++ b/infra/bots/isolate_win_toolchain.py
@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+import argparse
+import json
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+import utils
+
+
+REPO_CHROME = 'https://chromium.googlesource.com/chromium/src.git'
+REPO_SKIA = 'https://skia.googlesource.com/skia.git'
+
+
+def get_toolchain_dir(toolchain_dir_output):
+  """Find the toolchain directory."""
+  prefix = 'vs_path = '
+  for line in toolchain_dir_output.splitlines():
+    if line.startswith(prefix):
+      return line[len(prefix):].strip('"')
+  raise Exception('Unable to find toolchain dir in output:\n%s' % (
+                  toolchain_dir_output))
+
+
+def gen_toolchain(chrome_path, msvs_version, isolate_file):
+  """Update the VS toolchain, isolate it, and return the isolated hash."""
+  with utils.chdir(chrome_path):
+    subprocess.check_call([utils.GCLIENT, 'sync'])
+    with utils.git_branch():
+      vs_toolchain_py = os.path.join('build', 'vs_toolchain.py')
+      env = os.environ.copy()
+      env['GYP_MSVS_VERSION'] = msvs_version
+      subprocess.check_call(['python', vs_toolchain_py, 'update'], env=env)
+      output = subprocess.check_output(['python', vs_toolchain_py,
+                                        'get_toolchain_dir'], env=env).rstrip()
+      src_dir = get_toolchain_dir(output)
+
+    # Isolate the toolchain. Assumes we're running on Windows, since the above
+    # would fail otherwise.
+    rel_path = os.path.relpath(src_dir, os.path.dirname(isolate_file))
+    isolate = os.path.join(
+        os.curdir, 'tools', 'luci-go', 'win64', 'isolate.exe')
+    isolate_cmd = [isolate, 'archive', '--quiet',
+        '--isolate-server', 'https://isolateserver.appspot.com',
+        '-i', isolate_file,
+        '-s', 'win_toolchain_%s.isolated' % msvs_version,
+        '--extra-variable', 'WIN_TOOLCHAIN_DIR=%s' % rel_path]
+    isolate_out = subprocess.check_output(isolate_cmd).rstrip()
+    return shlex.split(isolate_out)[0]
+
+
+def update_toolchain_file(skia_path, msvs_version, isolated_hash):
+  """Edit the win_toolchain_hash file, upload a CL."""
+  with utils.chdir(skia_path):
+    with utils.git_branch():
+      hash_file = os.path.join('infra', 'bots', 'win_toolchain_hash.json')
+      with open(hash_file) as f:
+        hashes = json.load(f)
+      hashes[msvs_version] = isolated_hash
+      with open(hash_file, 'w') as f:
+        json.dump(hashes, f, indent=4, sort_keys=True)
+      subprocess.check_call([utils.GIT, 'add', hash_file])
+      subprocess.check_call([utils.GIT, 'commit', '-m', 'Update Win toolchain'])
+      subprocess.check_call([utils.GIT, 'cl', 'upload', '--bypass-hooks'])
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--msvs_version', required=True)
+  parser.add_argument('--chrome_path')
+  parser.add_argument('--skia_path')
+  args = parser.parse_args()
+
+  isolate_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                              'win_toolchain.isolate')
+
+  with utils.print_timings():
+    with utils.tmp_dir() as tmp_dir:
+      chrome_path = args.chrome_path
+      if not chrome_path:
+        print ('Syncing Chrome from scratch. If you already have a checkout, '
+               'specify --chrome_path to save time.')
+        chrome_path = os.path.join(tmp_dir.name, 'src')
+      if not os.path.isdir(chrome_path):
+        utils.git_clone(REPO_CHROME, chrome_path)
+
+      skia_path = args.skia_path
+      if not skia_path:
+        print ('Syncing Skia from scratch. If you already have a checkout, '
+               'specify --chrome_path to save time.')
+        skia_path = os.path.join(tmp_dir.name, 'skia')
+      if not os.path.isdir(skia_path):
+        utils.git_clone(REPO_SKIA, skia_path)
+
+      isolated_hash = gen_toolchain(chrome_path, args.msvs_version,
+                                    isolate_file)
+      update_toolchain_file(skia_path, args.msvs_version, isolated_hash)
+
+
+if __name__ == '__main__':
+  main()
diff --git a/infra/bots/utils.py b/infra/bots/utils.py
index ee648a26a4..a60ebe97b8 100644
--- a/infra/bots/utils.py
+++ b/infra/bots/utils.py
@@ -7,6 +7,18 @@
 
 
 import datetime
+import errno
+import os
+import shutil
+import sys
+import subprocess
+import tempfile
+import time
+import uuid
+
+
+GCLIENT = 'gclient.bat' if sys.platform == 'win32' else 'gclient'
+GIT = 'git.bat' if sys.platform == 'win32' else 'git'
 
 
 class print_timings(object):
@@ -21,3 +33,168 @@ class print_timings(object):
     finish = datetime.datetime.utcnow()
     duration = (finish-self._start).total_seconds()
     print 'Task finished at %s GMT (%f seconds)' % (str(finish), duration)
+
+
+class tmp_dir(object):
+  """Helper class used for creating a temporary directory and working in it."""
+  def __init__(self):
+    self._orig_dir = None
+    self._tmp_dir = None
+
+  def __enter__(self):
+    self._orig_dir = os.getcwd()
+    self._tmp_dir = tempfile.mkdtemp()
+    os.chdir(self._tmp_dir)
+    return self
+
+  def __exit__(self, t, v, tb):
+    os.chdir(self._orig_dir)
+    RemoveDirectory(self._tmp_dir)
+
+  @property
+  def name(self):
+    return self._tmp_dir
+
+
+class chdir(object):
+  """Helper class used for changing into and out of a directory."""
+  def __init__(self, d):
+    self._dir = d
+    self._orig_dir = None
+
+  def __enter__(self):
+    self._orig_dir = os.getcwd()
+    os.chdir(self._dir)
+    return self
+
+  def __exit__(self, t, v, tb):
+    os.chdir(self._orig_dir)
+
+
+def git_clone(repo_url, dest_dir):
+  """Clone the given repo into the given destination directory."""
+  subprocess.check_call([GIT, 'clone', repo_url, dest_dir])
+
+
+class git_branch(object):
+  """Check out a temporary git branch.
+
+  On exit, deletes the branch and attempts to restore the original state.
+  """
+  def __init__(self):
+    self._branch = None
+    self._orig_branch = None
+    self._stashed = False
+
+  def __enter__(self):
+    output = subprocess.check_output([GIT, 'stash'])
+    self._stashed = 'No local changes' not in output
+
+    # Get the original branch name or commit hash.
+    self._orig_branch = subprocess.check_output([
+        GIT, 'rev-parse', '--abbrev-ref', 'HEAD']).rstrip()
+    if self._orig_branch == 'HEAD':
+      self._orig_branch = subprocess.check_output([
+          GIT, 'rev-parse', 'HEAD']).rstrip()
+
+    # Check out a new branch, based at updated origin/master.
+    subprocess.check_call([GIT, 'fetch', 'origin'])
+    self._branch = '_tmp_%s' % uuid.uuid4()
+    subprocess.check_call([GIT, 'checkout', '-b', self._branch,
+                           '-t', 'origin/master'])
+    return self
+
+  def __exit__(self, exc_type, _value, _traceback):
+    subprocess.check_call([GIT, 'reset', '--hard', 'HEAD'])
+    subprocess.check_call([GIT, 'checkout', self._orig_branch])
+    if self._stashed:
+      subprocess.check_call([GIT, 'stash', 'pop'])
+    subprocess.check_call([GIT, 'branch', '-D', self._branch])
+
+
+def RemoveDirectory(*path):
+  """Recursively removes a directory, even if it's marked read-only.
+
+  This was copied from:
+  https://chromium.googlesource.com/chromium/tools/build/+/f3e7ff03613cd59a463b2ccc49773c3813e77404/scripts/common/chromium_utils.py#491
+
+  Remove the directory located at *path, if it exists.
+
+  shutil.rmtree() doesn't work on Windows if any of the files or directories
+  are read-only, which svn repositories and some .svn files are.  We need to
+  be able to force the files to be writable (i.e., deletable) as we traverse
+  the tree.
+
+  Even with all this, Windows still sometimes fails to delete a file, citing
+  a permission error (maybe something to do with antivirus scans or disk
+  indexing).  The best suggestion any of the user forums had was to wait a
+  bit and try again, so we do that too.  It's hand-waving, but sometimes it
+  works. :/
+  """
+  file_path = os.path.join(*path)
+  if not os.path.exists(file_path):
+    return
+
+  if sys.platform == 'win32':
+    # Give up and use cmd.exe's rd command.
+    file_path = os.path.normcase(file_path)
+    for _ in xrange(3):
+      print 'RemoveDirectory running %s' % (' '.join(
+          ['cmd.exe', '/c', 'rd', '/q', '/s', file_path]))
+      if not subprocess.call(['cmd.exe', '/c', 'rd', '/q', '/s', file_path]):
+        break
+      print '  Failed'
+      time.sleep(3)
+    return
+
+  def RemoveWithRetry_non_win(rmfunc, path):
+    if os.path.islink(path):
+      return os.remove(path)
+    else:
+      return rmfunc(path)
+
+  remove_with_retry = RemoveWithRetry_non_win
+
+  def RmTreeOnError(function, path, excinfo):
+    r"""This works around a problem whereby python 2.x on Windows has no ability
+    to check for symbolic links.  os.path.islink always returns False.  But
+    shutil.rmtree will fail if invoked on a symbolic link whose target was
+    deleted before the link.  E.g., reproduce like this:
+    > mkdir test
+    > mkdir test\1
+    > mklink /D test\current test\1
+    > python -c "import chromium_utils; chromium_utils.RemoveDirectory('test')"
+    To avoid this issue, we pass this error-handling function to rmtree.  If
+    we see the exact sort of failure, we ignore it.  All other failures we re-
+    raise.
+    """
+
+    exception_type = excinfo[0]
+    exception_value = excinfo[1]
+    # If shutil.rmtree encounters a symbolic link on Windows, os.listdir will
+    # fail with a WindowsError exception with an ENOENT errno (i.e., file not
+    # found).  We'll ignore that error.  Note that WindowsError is not defined
+    # for non-Windows platforms, so we use OSError (of which it is a subclass)
+    # to avoid lint complaints about an undefined global on non-Windows
+    # platforms.
+    if (function is os.listdir) and issubclass(exception_type, OSError):
+      if exception_value.errno == errno.ENOENT:
+        # File does not exist, and we're trying to delete, so we can ignore the
+        # failure.
+        print 'WARNING:  Failed to list %s during rmtree.  Ignoring.\n' % path
+      else:
+        raise
+    else:
+      raise
+
+  for root, dirs, files in os.walk(file_path, topdown=False):
+    # For POSIX:  making the directory writable guarantees removability.
+    # Windows will ignore the non-read-only bits in the chmod value.
+    os.chmod(root, 0770)
+    for name in files:
+      remove_with_retry(os.remove, os.path.join(root, name))
+    for name in dirs:
+      remove_with_retry(lambda p: shutil.rmtree(p, onerror=RmTreeOnError),
+                        os.path.join(root, name))
+
+  remove_with_retry(os.rmdir, file_path)
diff --git a/infra/bots/win_toolchain.isolate b/infra/bots/win_toolchain.isolate
new file mode 100644
index 0000000000..d9505bab27
--- /dev/null
+++ b/infra/bots/win_toolchain.isolate
@@ -0,0 +1,7 @@
+{
+  'variables': {
+    'files': [
+      '<(WIN_TOOLCHAIN_DIR)/',
+    ],
+  },
+}
diff --git a/infra/bots/win_toolchain_hash.json b/infra/bots/win_toolchain_hash.json
new file mode 100644
index 0000000000..337fd4fe6e
--- /dev/null
+++ b/infra/bots/win_toolchain_hash.json
@@ -0,0 +1,4 @@
+{
+    "2013": "d6b889963f8a6896d03457c52392f7f97d6cb94c", 
+    "2015": "081ab2320c1f76e234696dd2fc8aab44fa569a8a"
+}
\ No newline at end of file