diff options
| -rw-r--r-- | DEPS | 2 | ||||
| -rw-r--r-- | PRESUBMIT.py | 28 | ||||
| -rwxr-xr-x | tools/add_codereview_message.py | 115 | ||||
| -rw-r--r-- | tools/fix_pythonpath.py | 24 | ||||
| -rwxr-xr-x | tools/git-sync-deps | 11 | ||||
| -rw-r--r-- | tools/git_utils.py | 168 | ||||
| -rw-r--r-- | tools/misc_utils.py | 170 | ||||
| -rwxr-xr-x | tools/roll_deps.py | 400 |
8 files changed, 153 insertions, 765 deletions
@@ -3,7 +3,7 @@ use_relative_paths = True # Dependencies on outside packages. # deps = { - "common": "https://skia.googlesource.com/common.git@9bda9ca2f55b584189b83457d1cfea7805713f64", + "common": "https://skia.googlesource.com/common.git@767bf85299d7a822796b57252e7de1e84e6b2e61", # DEPS using https://chromium.googlesource.com are pulled from chromium @ r205199 # (see https://chromium.googlesource.com/chromium/chromium/+/c59bfa8ef877f45bfa859669053859857af1d279) diff --git a/PRESUBMIT.py b/PRESUBMIT.py index dc8cb77c0c..81bb257b92 100644 --- a/PRESUBMIT.py +++ b/PRESUBMIT.py @@ -46,6 +46,31 @@ def _CheckChangeHasEol(input_api, output_api, source_file_filter=None): return [] +def _PythonChecks(input_api, output_api): + """Run checks on any modified Python files.""" + pylint_disabled_warnings = ( + 'F0401', # Unable to import. + 'E0611', # No name in module. + 'W0232', # Class has no __init__ method. + 'E1002', # Use of super on an old style class. + 'W0403', # Relative import used. + 'R0201', # Method could be a function. + 'E1003', # Using class name in super. + 'W0613', # Unused argument. + ) + # Run Pylint on only the modified python files. Unfortunately it still runs + # Pylint on the whole file instead of just the modified lines. + affected_python_files = [] + for affected_file in input_api.AffectedSourceFiles(None): + affected_file_path = affected_file.LocalPath() + if affected_file_path.endswith('.py'): + affected_python_files.append(affected_file_path) + return input_api.canned_checks.RunPylint( + input_api, output_api, + disabled_warnings=pylint_disabled_warnings, + white_list=affected_python_files) + + def _CommonChecks(input_api, output_api): """Presubmit checks common to upload and commit.""" results = [] @@ -58,6 +83,7 @@ def _CommonChecks(input_api, output_api): results.extend( _CheckChangeHasEol( input_api, output_api, source_file_filter=sources)) + results.extend(_PythonChecks(input_api, output_api)) return results @@ -189,7 +215,7 @@ def _CheckLGTMsForPublicAPI(input_api, output_api): 'lgtm' in message['text'].lower()): # Found an lgtm in a message from an owner. lgtm_from_owner = True - break; + break if not lgtm_from_owner: results.append( diff --git a/tools/add_codereview_message.py b/tools/add_codereview_message.py index 6710390ac1..a7c53a9a5e 100755 --- a/tools/add_codereview_message.py +++ b/tools/add_codereview_message.py @@ -7,14 +7,12 @@ """Add message to codereview issue. -This script takes a codereview URL or a codereview issue number as its -argument and a (possibly multi-line) message on stdin. It then calls -`git cl upload` to append the message to the given codereview issue. +This script takes a codereview issue number as its argument and a (possibly +multi-line) message on stdin. It appends the message to the given issue. Usage: - echo MESSAGE | %prog -c CHECKOUT_PATH CODEREVIEW_ISSUE + echo MESSAGE | %prog CODEREVIEW_ISSUE or: - cd /path/to/git/checkout %prog CODEREVIEW_ISSUE <<EOF MESSAGE EOF @@ -23,77 +21,29 @@ or: """ import optparse -import os import sys -import git_utils -import misc_utils +import fix_pythonpath # pylint: disable=W0611 +from common.py.utils import find_depot_tools # pylint: disable=W0611 +import rietveld -DEFAULT_REVIEWERS = ','.join([ - 'rmistry@google.com', - 'reed@google.com', - 'bsalomon@google.com', - 'robertphillips@google.com', - ]) +RIETVELD_URL = 'https://codereview.chromium.org' -DEFAULT_CC_LIST = ','.join([ - 'skia-team@google.com', - ]) - - -def add_codereview_message(codereview_url, message, checkout_path, - skip_cl_upload, verbose, reviewers, cclist): +def add_codereview_message(issue, message): """Add a message to a given codereview. Args: codereview_url: (string) we will extract the issue number from this url, or this could simply be the issue number. - message: (string) will be passed to `git cl upload -m $MESSAGE` - checkout_path: (string) location of the git - repository checkout to be used. - skip_cl_upload: (boolean) if true, don't actually - add the message and keep the temporary branch around. - verbose: (boolean) print out details useful for debugging. - reviewers: (string) comma-separated list of reviewers - cclist: (string) comma-separated list of addresses to be - carbon-copied + message: (string) message to add. """ - # pylint: disable=I0011,R0913 - git = git_utils.git_executable() - issue = codereview_url.strip('/').split('/')[-1] - vsp = misc_utils.VerboseSubprocess(verbose) - if skip_cl_upload: - branch_name = 'issue_%s' % issue - else: - branch_name = None - upstream = 'origin/master' - - with misc_utils.ChangeDir(checkout_path, verbose): - vsp.check_call([git, 'fetch', '-q', 'origin']) - - with git_utils.ChangeGitBranch(branch_name, upstream, verbose): - vsp.check_call([git, 'cl', 'patch', issue]) - - git_upload = [ - git, 'cl', 'upload', '-t', 'bot report', '-m', message] - if cclist: - git_upload.append('--cc=' + cclist) - if reviewers: - git_upload.append('--reviewers=' + reviewers) - - if skip_cl_upload: - branch_name = git_utils.git_branch_name(verbose) - space = ' ' - print 'You should call:' - misc_utils.print_subprocess_args(space, ['cd', os.getcwd()]) - misc_utils.print_subprocess_args( - space, [git, 'checkout', branch_name]) - misc_utils.print_subprocess_args(space, git_upload) - else: - vsp.check_call(git_upload) - print vsp.check_output([git, 'cl', 'issue']) + # Passing None for the email and password will result in a prompt or + # reuse of existing cached credentials. + my_rietveld = rietveld.Rietveld(RIETVELD_URL, email=None, password=None) + + my_rietveld.add_comment(issue, message) def main(argv): @@ -103,44 +53,15 @@ def main(argv): argv: sys.argv[1:]-type argument list. """ option_parser = optparse.OptionParser(usage=__doc__) - option_parser.add_option( - '-c', '--checkout_path', - default=os.curdir, - help='Path to the Git repository checkout,' - ' defaults to current working directory.') - option_parser.add_option( - '', '--skip_cl_upload', action='store_true', default=False, - help='Skip the cl upload step; useful for testing.') - option_parser.add_option( - '', '--verbose', action='store_true', dest='verbose', default=False, - help='Do not suppress the output from `git cl`.',) - option_parser.add_option( - '', '--git_path', default='git', - help='Git executable, defaults to "git".',) - option_parser.add_option( - '', '--reviewers', default=DEFAULT_REVIEWERS, - help=('Comma-separated list of reviewers. Default is "%s".' - % DEFAULT_REVIEWERS)) - option_parser.add_option( - '', '--cc', default=DEFAULT_CC_LIST, - help=('Comma-separated list of addresses to be carbon-copied.' - ' Default is "%s".' % DEFAULT_CC_LIST)) - - options, arguments = option_parser.parse_args(argv) - - if not options.checkout_path: - option_parser.error('Must specify checkout_path.') - if not git_utils.git_executable(): - option_parser.error('Invalid git executable.') + _, arguments = option_parser.parse_args(argv) + if len(arguments) > 1: option_parser.error('Extra arguments.') if len(arguments) != 1: - option_parser.error('Missing Codereview URL.') + option_parser.error('Missing issue number.') message = sys.stdin.read() - add_codereview_message(arguments[0], message, options.checkout_path, - options.skip_cl_upload, options.verbose, - options.reviewers, options.cc) + add_codereview_message(int(arguments[0]), message) if __name__ == '__main__': diff --git a/tools/fix_pythonpath.py b/tools/fix_pythonpath.py new file mode 100644 index 0000000000..c84738ac85 --- /dev/null +++ b/tools/fix_pythonpath.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# Copyright (c) 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +"""Add the checkout root to sys.path, provide mechanisms for adding others.""" + + +import os +import sys + + +CHECKOUT_ROOT = os.path.realpath(os.path.join( + os.path.dirname(os.path.abspath(__file__)), os.pardir)) + + +def add_to_pythonpath(path): + """Add the given directory to PYTHONPATH.""" + sys.path.append(path) + + +add_to_pythonpath(CHECKOUT_ROOT) + diff --git a/tools/git-sync-deps b/tools/git-sync-deps index ee37e631f7..ac4576ddc7 100755 --- a/tools/git-sync-deps +++ b/tools/git-sync-deps @@ -11,9 +11,6 @@ Args: An optional list of deps_os values. Environment Variables: - GIT_EXECUTABLE: path to "git" binary; if unset, will look for one of - ['git', 'git.exe', 'git.bat'] in your default path. - GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset, will use the file ../DEPS relative to this script's directory. @@ -35,7 +32,8 @@ import subprocess import sys import threading -from git_utils import git_executable +import fix_pythonpath +from common.py.utils.git_utils import GIT DEFAULT_DEPS_PATH = os.path.normpath( @@ -148,9 +146,6 @@ def git_sync_deps(deps_file_path, deps_os_list, verbose): Raises DepsError exception and git Exceptions. """ - git = git_executable() - assert git - deps_file_directory = os.path.dirname(deps_file_path) deps = parse_file_to_dict(deps_file_path) dependencies = deps['deps'].copy() @@ -172,7 +167,7 @@ def git_sync_deps(deps_file_path, deps_os_list, verbose): relative_directory = os.path.join(deps_file_directory, directory) list_of_arg_lists.append( - (git, repo, checkoutable, relative_directory, verbose)) + (GIT, repo, checkoutable, relative_directory, verbose)) multithread(git_checkout_to_directory, list_of_arg_lists) diff --git a/tools/git_utils.py b/tools/git_utils.py deleted file mode 100644 index a35c85e20e..0000000000 --- a/tools/git_utils.py +++ /dev/null @@ -1,168 +0,0 @@ -# Copyright 2014 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Module to host the ChangeGitBranch class and test_git_executable function. -""" - -import os -import subprocess - -import misc_utils - - -class ChangeGitBranch(object): - """Class to manage git branches. - - This class allows one to create a new branch in a repository based - off of a given commit, and restore the original tree state. - - Assumes current working directory is a git repository. - - Example: - with ChangeGitBranch(): - edit_files(files) - git_add(files) - git_commit() - git_format_patch('HEAD~') - # At this point, the repository is returned to its original - # state. - - Constructor Args: - branch_name: (string) if not None, the name of the branch to - use. If None, then use a temporary branch that will be - deleted. If the branch already exists, then a different - branch name will be created. Use git_branch_name() to - find the actual branch name used. - upstream_branch: (string) if not None, the name of the branch or - commit to branch from. If None, then use origin/master - verbose: (boolean) if true, makes debugging easier. - - Raises: - OSError: the git executable disappeared. - subprocess.CalledProcessError: git returned unexpected status. - Exception: if the given branch name exists, or if the repository - isn't clean on exit, or git can't be found. - """ - # pylint: disable=I0011,R0903,R0902 - - def __init__(self, - branch_name=None, - upstream_branch=None, - verbose=False): - # pylint: disable=I0011,R0913 - if branch_name: - self._branch_name = branch_name - self._delete_branch = False - else: - self._branch_name = 'ChangeGitBranchTempBranch' - self._delete_branch = True - - if upstream_branch: - self._upstream_branch = upstream_branch - else: - self._upstream_branch = 'origin/master' - - self._git = git_executable() - if not self._git: - raise Exception('Git can\'t be found.') - - self._stash = None - self._original_branch = None - self._vsp = misc_utils.VerboseSubprocess(verbose) - - def _has_git_diff(self): - """Return true iff repository has uncommited changes.""" - return bool(self._vsp.call([self._git, 'diff', '--quiet', 'HEAD'])) - - def _branch_exists(self, branch): - """Return true iff branch exists.""" - return 0 == self._vsp.call([self._git, 'show-ref', '--quiet', branch]) - - def __enter__(self): - git, vsp = self._git, self._vsp - - if self._branch_exists(self._branch_name): - i, branch_name = 0, self._branch_name - while self._branch_exists(branch_name): - i += 1 - branch_name = '%s_%03d' % (self._branch_name, i) - self._branch_name = branch_name - - self._stash = self._has_git_diff() - if self._stash: - vsp.check_call([git, 'stash', 'save']) - self._original_branch = git_branch_name(vsp.verbose) - vsp.check_call( - [git, 'checkout', '-q', '-b', - self._branch_name, self._upstream_branch]) - - def __exit__(self, etype, value, traceback): - git, vsp = self._git, self._vsp - - if self._has_git_diff(): - status = vsp.check_output([git, 'status', '-s']) - raise Exception('git checkout not clean:\n%s' % status) - vsp.check_call([git, 'checkout', '-q', self._original_branch]) - if self._stash: - vsp.check_call([git, 'stash', 'pop']) - if self._delete_branch: - assert self._original_branch != self._branch_name - vsp.check_call([git, 'branch', '-D', self._branch_name]) - - -def git_branch_name(verbose=False): - """Return a description of the current branch. - - Args: - verbose: (boolean) makes debugging easier - - Returns: - A string suitable for passing to `git checkout` later. - """ - git = git_executable() - vsp = misc_utils.VerboseSubprocess(verbose) - try: - full_branch = vsp.strip_output([git, 'symbolic-ref', 'HEAD']) - return full_branch.split('/')[-1] - except (subprocess.CalledProcessError,): - # "fatal: ref HEAD is not a symbolic ref" - return vsp.strip_output([git, 'rev-parse', 'HEAD']) - - -def test_git_executable(git): - """Test the git executable. - - Args: - git: git executable path. - Returns: - True if test is successful. - """ - with open(os.devnull, 'w') as devnull: - try: - subprocess.call([git, '--version'], stdout=devnull) - except (OSError,): - return False - return True - - -def git_executable(): - """Find the git executable. - - If the GIT_EXECUTABLE environment variable is set, that will - override whatever is found in the PATH. - - If no suitable executable is found, return None - - Returns: - A string suiable for passing to subprocess functions, or None. - """ - env_git = os.environ.get('GIT_EXECUTABLE') - if env_git and test_git_executable(env_git): - return env_git - for git in ('git', 'git.exe', 'git.bat'): - if test_git_executable(git): - return git - return None - diff --git a/tools/misc_utils.py b/tools/misc_utils.py index 13978a4742..c3e09da40f 100644 --- a/tools/misc_utils.py +++ b/tools/misc_utils.py @@ -4,153 +4,10 @@ # found in the LICENSE file. -"""Module to host the VerboseSubprocess, ChangeDir, and ReSearch classes. -""" +"""Miscellaneous utilities.""" -import os -import re -import subprocess - - -def print_subprocess_args(prefix, *args, **kwargs): - """Print out args in a human-readable manner.""" - def quote_and_escape(string): - """Quote and escape a string if necessary.""" - if ' ' in string or '\n' in string: - string = '"%s"' % string.replace('"', '\\"') - return string - if 'cwd' in kwargs: - print '%scd %s' % (prefix, kwargs['cwd']) - print prefix + ' '.join(quote_and_escape(arg) for arg in args[0]) - if 'cwd' in kwargs: - print '%scd -' % prefix - - -class VerboseSubprocess(object): - """Call subprocess methods, but print out command before executing. - - Attributes: - verbose: (boolean) should we print out the command or not. If - not, this is the same as calling the subprocess method - quiet: (boolean) suppress stdout on check_call and call. - prefix: (string) When verbose, what to print before each command. - """ - - def __init__(self, verbose): - self.verbose = verbose - self.quiet = not verbose - self.prefix = '~~$ ' - - def check_call(self, *args, **kwargs): - """Wrapper for subprocess.check_call(). - - Args: - *args: to be passed to subprocess.check_call() - **kwargs: to be passed to subprocess.check_call() - Returns: - Whatever subprocess.check_call() returns. - Raises: - OSError or subprocess.CalledProcessError: raised by check_call. - """ - if self.verbose: - print_subprocess_args(self.prefix, *args, **kwargs) - if self.quiet: - with open(os.devnull, 'w') as devnull: - return subprocess.check_call(*args, stdout=devnull, **kwargs) - else: - return subprocess.check_call(*args, **kwargs) - - def call(self, *args, **kwargs): - """Wrapper for subprocess.check(). - - Args: - *args: to be passed to subprocess.check_call() - **kwargs: to be passed to subprocess.check_call() - Returns: - Whatever subprocess.call() returns. - Raises: - OSError or subprocess.CalledProcessError: raised by call. - """ - if self.verbose: - print_subprocess_args(self.prefix, *args, **kwargs) - if self.quiet: - with open(os.devnull, 'w') as devnull: - return subprocess.call(*args, stdout=devnull, **kwargs) - else: - return subprocess.call(*args, **kwargs) - - def check_output(self, *args, **kwargs): - """Wrapper for subprocess.check_output(). - - Args: - *args: to be passed to subprocess.check_output() - **kwargs: to be passed to subprocess.check_output() - Returns: - Whatever subprocess.check_output() returns. - Raises: - OSError or subprocess.CalledProcessError: raised by check_output. - """ - if self.verbose: - print_subprocess_args(self.prefix, *args, **kwargs) - return subprocess.check_output(*args, **kwargs) - - def strip_output(self, *args, **kwargs): - """Wrap subprocess.check_output and str.strip(). - - Pass the given arguments into subprocess.check_output() and return - the results, after stripping any excess whitespace. - - Args: - *args: to be passed to subprocess.check_output() - **kwargs: to be passed to subprocess.check_output() - Returns: - The output of the process as a string without leading or - trailing whitespace. - Raises: - OSError or subprocess.CalledProcessError: raised by check_output. - """ - if self.verbose: - print_subprocess_args(self.prefix, *args, **kwargs) - return str(subprocess.check_output(*args, **kwargs)).strip() - - def popen(self, *args, **kwargs): - """Wrapper for subprocess.Popen(). - - Args: - *args: to be passed to subprocess.Popen() - **kwargs: to be passed to subprocess.Popen() - Returns: - The output of subprocess.Popen() - Raises: - OSError or subprocess.CalledProcessError: raised by Popen. - """ - if self.verbose: - print_subprocess_args(self.prefix, *args, **kwargs) - return subprocess.Popen(*args, **kwargs) - - -class ChangeDir(object): - """Use with a with-statement to temporarily change directories.""" - # pylint: disable=I0011,R0903 - - def __init__(self, directory, verbose=False): - self._directory = directory - self._verbose = verbose - - def __enter__(self): - if self._directory != os.curdir: - if self._verbose: - print '~~$ cd %s' % self._directory - cwd = os.getcwd() - os.chdir(self._directory) - self._directory = cwd - - def __exit__(self, etype, value, traceback): - if self._directory != os.curdir: - if self._verbose: - print '~~$ cd %s' % self._directory - os.chdir(self._directory) +import re class ReSearch(object): @@ -199,26 +56,3 @@ class ReSearch(object): """ match = re.search(pattern, input_string) return match.group('return') if match else default - - @staticmethod - def search_within_output(verbose, pattern, default, *args, **kwargs): - """Search for regular expression in a process output. - - Does not search across newlines. - - Args: - verbose: (boolean) shoule we call print_subprocess_args? - pattern: (string) to be passed to re.compile - default: what to return if no match - *args: to be passed to subprocess.Popen() - **kwargs: to be passed to subprocess.Popen() - - Returns: - A string or whatever default is - """ - if verbose: - print_subprocess_args('~~$ ', *args, **kwargs) - proc = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs) - return ReSearch.search_within_stream(proc.stdout, pattern, default) - - diff --git a/tools/roll_deps.py b/tools/roll_deps.py index d280bda11e..18f4157bae 100755 --- a/tools/roll_deps.py +++ b/tools/roll_deps.py @@ -28,12 +28,13 @@ import optparse import os import re import shutil -import subprocess import sys import tempfile -import git_utils -import misc_utils +import fix_pythonpath # pylint: disable=W0611 +from common.py.utils import git_utils +from common.py.utils import misc +from common.py.utils import shell_utils DEFAULT_BOTS_LIST = [ @@ -60,41 +61,32 @@ DEFAULT_BOTS_LIST = [ 'win_layout_rel', ] +REGEXP_SKIA_REVISION = ( + r'^ "skia_revision": "(?P<revision>[0-9a-fA-F]{2,40})",$') + class DepsRollConfig(object): """Contains configuration options for this module. Attributes: - git: (string) The git executable. chromium_path: (string) path to a local chromium git repository. save_branches: (boolean) iff false, delete temporary branches. verbose: (boolean) iff false, suppress the output from git-cl. - search_depth: (int) how far back to look for the revision. - skia_url: (string) Skia's git repository. - self.skip_cl_upload: (boolean) - self.cl_bot_list: (list of strings) + skip_cl_upload: (boolean) + cl_bot_list: (list of strings) """ # pylint: disable=I0011,R0903,R0902 def __init__(self, options=None): - self.skia_url = 'https://skia.googlesource.com/skia.git' - self.revision_format = ( - 'git-svn-id: http://skia.googlecode.com/svn/trunk@%d ') - - self.git = git_utils.git_executable() - if not options: options = DepsRollConfig.GetOptionParser() # pylint: disable=I0011,E1103 self.verbose = options.verbose - self.vsp = misc_utils.VerboseSubprocess(self.verbose) self.save_branches = not options.delete_branches - self.search_depth = options.search_depth self.chromium_path = options.chromium_path self.skip_cl_upload = options.skip_cl_upload # Split and remove empty strigns from the bot list. self.cl_bot_list = [bot for bot in options.bots.split(',') if bot] - self.skia_git_checkout_path = options.skia_git_path self.default_branch_name = 'autogenerated_deps_roll_branch' self.reviewers_list = ','.join([ # 'rmistry@google.com', @@ -125,23 +117,9 @@ class DepsRollConfig(object): ' if that environment variable is set.', default=os.environ.get('CHROMIUM_CHECKOUT_PATH')) option_parser.add_option( - '-r', '--revision', type='int', default=None, - help='The Skia SVN revision number, defaults to top of tree.') - option_parser.add_option( - '-g', '--git_hash', default=None, - help='A partial Skia Git hash. Do not set this and revision.') + '-r', '--revision', default=None, + help='The Skia Git commit hash.') - # Anyone using this script on a regular basis should set the - # SKIA_GIT_CHECKOUT_PATH environment variable. - option_parser.add_option( - '', '--skia_git_path', - help='Path of a pure-git Skia repository checkout. If empty,' - ' a temporary will be cloned. Defaults to SKIA_GIT_CHECKOUT' - '_PATH, if that environment variable is set.', - default=os.environ.get('SKIA_GIT_CHECKOUT_PATH')) - option_parser.add_option( - '', '--search_depth', type='int', default=100, - help='How far back to look for the revision.') option_parser.add_option( '', '--delete_branches', help='Delete the temporary branches', action='store_true', dest='delete_branches', default=False) @@ -169,236 +147,58 @@ class DepsRollError(Exception): pass -def get_svn_revision(config, commit): - """Works in both git and git-svn. returns a string.""" - svn_format = ( - '(git-svn-id: [^@ ]+@|SVN changes up to revision |' - 'LKGR w/ DEPS up to revision )(?P<return>[0-9]+)') - svn_revision = misc_utils.ReSearch.search_within_output( - config.verbose, svn_format, None, - [config.git, 'log', '-n', '1', '--format=format:%B', commit]) - if not svn_revision: - raise DepsRollError( - 'Revision number missing from Chromium origin/master.') - return int(svn_revision) - - -class SkiaGitCheckout(object): - """Class to create a temporary skia git checkout, if necessary. - """ - # pylint: disable=I0011,R0903 - - def __init__(self, config, depth): - self._config = config - self._depth = depth - self._use_temp = None - self._original_cwd = None - - def __enter__(self): - config = self._config - git = config.git - skia_dir = None - self._original_cwd = os.getcwd() - if config.skia_git_checkout_path: - if config.skia_git_checkout_path != os.curdir: - skia_dir = config.skia_git_checkout_path - ## Update origin/master if needed. - if self._config.verbose: - print '~~$', 'cd', skia_dir - os.chdir(skia_dir) - config.vsp.check_call([git, 'fetch', '-q', 'origin']) - self._use_temp = None - else: - skia_dir = tempfile.mkdtemp(prefix='git_skia_tmp_') - self._use_temp = skia_dir - try: - os.chdir(skia_dir) - config.vsp.check_call( - [git, 'clone', '-q', '--depth=%d' % self._depth, - '--single-branch', config.skia_url, '.']) - except (OSError, subprocess.CalledProcessError) as error: - shutil.rmtree(skia_dir) - raise error - - def __exit__(self, etype, value, traceback): - if self._config.skia_git_checkout_path != os.curdir: - if self._config.verbose: - print '~~$', 'cd', self._original_cwd - os.chdir(self._original_cwd) - if self._use_temp: - shutil.rmtree(self._use_temp) - - -def revision_and_hash(config): - """Finds revision number and git hash of origin/master in the Skia tree. - - Args: - config: (roll_deps.DepsRollConfig) object containing options. - - Returns: - A tuple (revision, hash) - revision: (int) SVN revision number. - git_hash: (string) full Git commit hash. - - Raises: - roll_deps.DepsRollError: if the revision can't be found. - OSError: failed to execute git or git-cl. - subprocess.CalledProcessError: git returned unexpected status. - """ - with SkiaGitCheckout(config, 1): - revision = get_svn_revision(config, 'origin/master') - git_hash = config.vsp.strip_output( - [config.git, 'show-ref', 'origin/master', '--hash']) - if not git_hash: - raise DepsRollError('Git hash can not be found.') - return revision, git_hash - - -def revision_and_hash_from_revision(config, revision): - """Finds revision number and git hash of a commit in the Skia tree. - - Args: - config: (roll_deps.DepsRollConfig) object containing options. - revision: (int) SVN revision number. - - Returns: - A tuple (revision, hash) - revision: (int) SVN revision number. - git_hash: (string) full Git commit hash. - - Raises: - roll_deps.DepsRollError: if the revision can't be found. - OSError: failed to execute git or git-cl. - subprocess.CalledProcessError: git returned unexpected status. - """ - with SkiaGitCheckout(config, config.search_depth): - revision_regex = config.revision_format % revision - git_hash = config.vsp.strip_output( - [config.git, 'log', '--grep', revision_regex, - '--format=format:%H', 'origin/master']) - if not git_hash: - raise DepsRollError('Git hash can not be found.') - return revision, git_hash - - -def revision_and_hash_from_partial(config, partial_hash): - """Returns the SVN revision number and full git hash. - - Args: - config: (roll_deps.DepsRollConfig) object containing options. - partial_hash: (string) Partial git commit hash. - - Returns: - A tuple (revision, hash) - revision: (int) SVN revision number. - git_hash: (string) full Git commit hash. - - Raises: - roll_deps.DepsRollError: if the revision can't be found. - OSError: failed to execute git or git-cl. - subprocess.CalledProcessError: git returned unexpected status. - """ - with SkiaGitCheckout(config, config.search_depth): - git_hash = config.vsp.strip_output( - ['git', 'log', '-n', '1', '--format=format:%H', partial_hash]) - if not git_hash: - raise DepsRollError('Partial Git hash can not be found.') - revision = get_svn_revision(config, git_hash) - return revision, git_hash - - -def change_skia_deps(revision, git_hash, depspath): +def change_skia_deps(revision, depspath): """Update the DEPS file. - Modify the skia_revision and skia_hash entries in the given DEPS file. + Modify the skia_revision entry in the given DEPS file. Args: - revision: (int) Skia SVN revision. - git_hash: (string) Skia Git hash. + revision: (string) Skia commit hash. depspath: (string) path to DEPS file. """ temp_file = tempfile.NamedTemporaryFile(delete=False, prefix='skia_DEPS_ROLL_tmp_') try: - deps_regex_rev = re.compile('"skia_revision": "[0-9]*",') - deps_regex_hash = re.compile('"skia_hash": "[0-9a-f]*",') - - deps_regex_rev_repl = '"skia_revision": "%d",' % revision - deps_regex_hash_repl = '"skia_hash": "%s",' % git_hash + deps_regex_rev = re.compile(REGEXP_SKIA_REVISION) + deps_regex_rev_repl = ' "skia_revision": "%s",' % revision with open(depspath, 'r') as input_stream: for line in input_stream: line = deps_regex_rev.sub(deps_regex_rev_repl, line) - line = deps_regex_hash.sub(deps_regex_hash_repl, line) temp_file.write(line) finally: temp_file.close() shutil.move(temp_file.name, depspath) -def git_cl_uploader(config, message, file_list): - """Create a commit in the current git branch; upload via git-cl. - - Assumes that you are already on the branch you want to be on. +def submit_tries(bots_to_run, dry_run=False): + """Submit try requests for the current branch on the given bots. Args: - config: (roll_deps.DepsRollConfig) object containing options. - message: (string) the commit message, can be multiline. - file_list: (list of strings) list of filenames to pass to `git add`. - - Returns: - The output of `git cl issue`, if not config.skip_cl_upload, else ''. + bots_to_run: (list of strings) bots to request. + dry_run: (bool) whether to actually submit the try request. """ - - git, vsp = config.git, config.vsp - svn_info = str(get_svn_revision(config, 'HEAD')) - - for filename in file_list: - assert os.path.exists(filename) - vsp.check_call([git, 'add', filename]) - - vsp.check_call([git, 'commit', '-q', '-m', message]) - - git_cl = [git, 'cl', 'upload', '-f', - '--bypass-hooks', '--bypass-watchlists'] - if config.cc_list: - git_cl.append('--cc=%s' % config.cc_list) - if config.reviewers_list: - git_cl.append('--reviewers=%s' % config.reviewers_list) - git_try = [ - git, 'cl', 'try', '-m', 'tryserver.chromium', '--revision', svn_info] - git_try.extend([arg for bot in config.cl_bot_list for arg in ('-b', bot)]) - - branch_name = git_utils.git_branch_name(vsp.verbose) + git_utils.GIT, 'cl', 'try', '-m', 'tryserver.chromium'] + git_try.extend([arg for bot in bots_to_run for arg in ('-b', bot)]) - if config.skip_cl_upload: + if dry_run: space = ' ' print 'You should call:' - print '%scd %s' % (space, os.getcwd()) - misc_utils.print_subprocess_args(space, [git, 'checkout', branch_name]) - misc_utils.print_subprocess_args(space, git_cl) - if config.cl_bot_list: - misc_utils.print_subprocess_args(space, git_try) + print space, git_try print - return '' else: - vsp.check_call(git_cl) - issue = vsp.strip_output([git, 'cl', 'issue']) - if config.cl_bot_list: - vsp.check_call(git_try) - return issue + shell_utils.run(git_try) -def roll_deps(config, revision, git_hash): +def roll_deps(config, revision): """Upload changed DEPS and a whitespace change. Given the correct git_hash, create two Reitveld issues. Args: config: (roll_deps.DepsRollConfig) object containing options. - revision: (int) Skia SVN revision. - git_hash: (string) Skia Git hash. + revision: (string) Skia Git hash. Returns: a tuple containing textual description of the two issues. @@ -408,115 +208,66 @@ def roll_deps(config, revision, git_hash): subprocess.CalledProcessError: git returned unexpected status. """ - git = config.git - with misc_utils.ChangeDir(config.chromium_path, config.verbose): - config.vsp.check_call([git, 'fetch', '-q', 'origin']) - - old_revision = misc_utils.ReSearch.search_within_output( - config.verbose, '"skia_revision": "(?P<return>[0-9]+)",', None, - [git, 'show', 'origin/master:DEPS']) + with misc.ChDir(config.chromium_path, verbose=config.verbose): + git_utils.Fetch() + output = shell_utils.run([git_utils.GIT, 'show', 'origin/master:DEPS'], + log_in_real_time=False).rstrip() + match = re.search(REGEXP_SKIA_REVISION, output, flags=re.MULTILINE) + old_revision = None + if match: + old_revision = match.group('revision') assert old_revision - if revision == int(old_revision): - print 'DEPS is up to date!' - return (None, None) - master_hash = config.vsp.strip_output( - [git, 'show-ref', 'origin/master', '--hash']) - master_revision = get_svn_revision(config, 'origin/master') + master_hash = git_utils.FullHash('origin/master').rstrip() # master_hash[8] gives each whitespace CL a unique name. - if config.save_branches: - branch = 'control_%s' % master_hash[:8] - else: - branch = None + branch = 'control_%s' % master_hash[:8] message = ('whitespace change %s\n\n' - 'Chromium base revision: %d / %s\n\n' + 'Chromium base revision: %s\n\n' 'This CL was created by Skia\'s roll_deps.py script.\n' - ) % (master_hash[:8], master_revision, master_hash[:8]) - with git_utils.ChangeGitBranch(branch, 'origin/master', - config.verbose): - branch = git_utils.git_branch_name(config.vsp.verbose) - - with open('build/whitespace_file.txt', 'a') as output_stream: - output_stream.write('\nCONTROL\n') - - whitespace_cl = git_cl_uploader( - config, message, ['build/whitespace_file.txt']) - - control_url = misc_utils.ReSearch.search_within_string( - whitespace_cl, '(?P<return>https?://[^) ]+)', '?') + ) % (master_hash[:8], master_hash[:8]) + with git_utils.GitBranch(branch, message, + delete_when_finished=not config.save_branches, + upload=not config.skip_cl_upload + ) as whitespace_branch: + branch = git_utils.GetCurrentBranch() + with open(os.path.join('build', 'whitespace_file.txt'), 'a') as f: + f.write('\nCONTROL\n') + + control_url = whitespace_branch.commit_and_upload() + if config.cl_bot_list: + submit_tries(config.cl_bot_list, dry_run=config.skip_cl_upload) + whitespace_cl = control_url if config.save_branches: - whitespace_cl = '%s\n branch: %s' % (whitespace_cl, branch) + whitespace_cl += '\n branch: %s' % branch - if config.save_branches: - branch = 'roll_%d_%s' % (revision, master_hash[:8]) - else: - branch = None + branch = 'roll_%s_%s' % (revision, master_hash[:8]) message = ( - 'roll skia DEPS to %d\n\n' - 'Chromium base revision: %d / %s\n' + 'roll skia DEPS to %s\n\n' + 'Chromium base revision: %s\n' 'Old Skia revision: %s\n' - 'New Skia revision: %d\n' + 'New Skia revision: %s\n' 'Control CL: %s\n\n' 'This CL was created by Skia\'s roll_deps.py script.\n\n' 'Bypassing commit queue trybots:\n' 'NOTRY=true\n' - % (revision, master_revision, master_hash[:8], - old_revision, revision, control_url)) - with git_utils.ChangeGitBranch(branch, 'origin/master', - config.verbose): - branch = git_utils.git_branch_name(config.vsp.verbose) - - change_skia_deps(revision, git_hash, 'DEPS') - deps_cl = git_cl_uploader(config, message, ['DEPS']) + % (revision, master_hash[:8], + old_revision[:8], revision[:8], control_url)) + with git_utils.GitBranch(branch, message, + delete_when_finished=not config.save_branches, + upload=not config.skip_cl_upload + ) as roll_branch: + change_skia_deps(revision, 'DEPS') + deps_url = roll_branch.commit_and_upload() + if config.cl_bot_list: + submit_tries(config.cl_bot_list, dry_run=config.skip_cl_upload) + deps_cl = deps_url if config.save_branches: - deps_cl = '%s\n branch: %s' % (deps_cl, branch) + deps_cl += '\n branch: %s' % branch return deps_cl, whitespace_cl -def find_hash_and_roll_deps(config, revision=None, partial_hash=None): - """Call find_hash_from_revision() and roll_deps(). - - The calls to git will be verbose on standard output. After a - successful upload of both issues, print links to the new - codereview issues. - - Args: - config: (roll_deps.DepsRollConfig) object containing options. - revision: (int or None) the Skia SVN revision number or None - to use the tip of the tree. - partial_hash: (string or None) a partial pure-git Skia commit - hash. Don't pass both partial_hash and revision. - - Raises: - roll_deps.DepsRollError: if the revision can't be found. - OSError: failed to execute git or git-cl. - subprocess.CalledProcessError: git returned unexpected status. - """ - - if revision and partial_hash: - raise DepsRollError('Pass revision or partial_hash, not both.') - - if partial_hash: - revision, git_hash = revision_and_hash_from_partial( - config, partial_hash) - elif revision: - revision, git_hash = revision_and_hash_from_revision(config, revision) - else: - revision, git_hash = revision_and_hash(config) - - print 'revision=%r\nhash=%r\n' % (revision, git_hash) - - deps_issue, whitespace_issue = roll_deps(config, revision, git_hash) - - if deps_issue and whitespace_issue: - print 'DEPS roll:\n %s\n' % deps_issue - print 'Whitespace change:\n %s\n' % whitespace_issue - else: - print >> sys.stderr, 'No issues created.' - - def main(args): """main function; see module-level docstring and GetOptionParser help. @@ -526,18 +277,23 @@ def main(args): option_parser = DepsRollConfig.GetOptionParser() options = option_parser.parse_args(args)[0] + if not options.revision: + option_parser.error('Must specify a revision.') if not options.chromium_path: option_parser.error('Must specify chromium_path.') if not os.path.isdir(options.chromium_path): option_parser.error('chromium_path must be a directory.') - if not git_utils.git_executable(): - option_parser.error('Invalid git executable.') - config = DepsRollConfig(options) - find_hash_and_roll_deps(config, options.revision, options.git_hash) + shell_utils.VERBOSE = options.verbose + deps_issue, whitespace_issue = roll_deps(config, options.revision) + + if deps_issue and whitespace_issue: + print 'DEPS roll:\n %s\n' % deps_issue + print 'Whitespace change:\n %s\n' % whitespace_issue + else: + print >> sys.stderr, 'No issues created.' if __name__ == '__main__': main(sys.argv[1:]) - |