diff options
| -rw-r--r-- | DEPS | 8 | ||||
| -rwxr-xr-x | gm/rebaseline_server/download_actuals.py | 72 | ||||
| -rwxr-xr-x | gm/rebaseline_server/server.py | 2 |
3 files changed, 68 insertions, 14 deletions
@@ -1,11 +1,6 @@ use_relative_paths = True -# Dependencies on outside packages, as needed for developers/bots that use -# "gclient" instead of raw SVN access. -# -# For now, this must be maintained in parallel with "SVN externals" -# dependencies for developers who use raw SVN instead of "gclient". -# See third_party/externals/README +# Dependencies on outside packages. # deps = { # DEPS using https://chromium.googlesource.com are pulled from chromium @ r205199 @@ -13,6 +8,7 @@ deps = { "third_party/externals/angle" : "https://chromium.googlesource.com/external/angleproject.git", "third_party/externals/angle2" : "https://chromium.googlesource.com/angle/angle.git@bdc9b2f0ed9e365bf5a4d19799d93a512f07dd32", "third_party/externals/freetype" : "https://skia.googlesource.com/third_party/freetype2.git@VER-2-5-0-1", + "third_party/externals/google-api-python-client" : "https://github.com/google/google-api-python-client.git@56557e2c1d2cbce0d2de26e3a7f32f836b8f5eb2", "third_party/externals/gyp" : "https://chromium.googlesource.com/external/gyp.git@5917c6a6b77c9e97a0cbb66847194381bd36ec4c", "third_party/externals/libjpeg" : "https://chromium.googlesource.com/chromium/deps/libjpeg_turbo.git@82ce8a6d4ebe12a177c0c3597192f2b4f09e81c3", "third_party/externals/jsoncpp" : "https://chromium.googlesource.com/external/jsoncpp/jsoncpp.git@ab1e40f3bce061ea6f9bdc60351d6cde2a4f872b", diff --git a/gm/rebaseline_server/download_actuals.py b/gm/rebaseline_server/download_actuals.py index 3f3f640f12..636958be6b 100755 --- a/gm/rebaseline_server/download_actuals.py +++ b/gm/rebaseline_server/download_actuals.py @@ -41,8 +41,17 @@ if TOOLS_DIRECTORY not in sys.path: import buildbot_globals import gm_json -DEFAULT_ACTUALS_BASE_URL = posixpath.join( - buildbot_globals.Get('autogen_svn_url'), 'gm-actual') +# Imports from third-party code +APICLIENT_DIRECTORY = os.path.join( + TRUNK_DIRECTORY, 'third_party', 'externals', 'google-api-python-client') +if APICLIENT_DIRECTORY not in sys.path: + sys.path.append(APICLIENT_DIRECTORY) +from googleapiclient.discovery import build as build_service + + +GM_SUMMARIES_BUCKET = buildbot_globals.Get('gm_summaries_bucket') +DEFAULT_ACTUALS_BASE_URL = ( + 'http://storage.googleapis.com/%s' % GM_SUMMARIES_BUCKET) DEFAULT_JSON_FILENAME = 'actual-results.json' @@ -96,6 +105,8 @@ class Download(object): test_name=test, hash_type=hash_type, hash_digest=hash_digest, gm_actuals_root_url=self._gm_actuals_root_url) dest_path = os.path.join(dest_dir, config, test + '.png') + # TODO(epoger): To speed this up, we should only download files that + # we don't already have on local disk. copy_contents(source_url=source_url, dest_path=dest_path, create_subdirs_if_needed=True) @@ -151,6 +162,43 @@ def copy_contents(source_url, dest_path, create_subdirs_if_needed=False): shutil.copyfileobj(fsrc=source_handle, fdst=dest_handle) +def gcs_list_bucket_contents(bucket, subdir=None): + """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple. + + Uses the API documented at + https://developers.google.com/storage/docs/json_api/v1/objects/list + + Args: + bucket: name of the Google Storage bucket + subdir: directory within the bucket to list, or None for root directory + """ + # The GCS command relies on the subdir name (if any) ending with a slash. + if subdir and not subdir.endswith('/'): + subdir += '/' + subdir_length = len(subdir) if subdir else 0 + + storage = build_service('storage', 'v1') + command = storage.objects().list( + bucket=bucket, delimiter='/', fields='items(name),prefixes', + prefix=subdir) + results = command.execute() + + # The GCS command returned two subdicts: + # prefixes: the full path of every directory within subdir, with trailing '/' + # items: property dict for each file object within subdir + # (including 'name', which is full path of the object) + dirs = [] + for dir_fullpath in results.get('prefixes', []): + dir_basename = dir_fullpath[subdir_length:] + dirs.append(dir_basename[:-1]) # strip trailing slash + files = [] + for file_properties in results.get('items', []): + file_fullpath = file_properties['name'] + file_basename = file_fullpath[subdir_length:] + files.append(file_basename) + return (dirs, files) + + def main(): parser = optparse.OptionParser() required_params = [] @@ -159,16 +207,17 @@ def main(): default=DEFAULT_ACTUALS_BASE_URL, help=('Base URL from which to read files containing JSON ' 'summaries of actual GM results; defaults to ' - '"%default". To get a specific revision (useful for ' - 'trybots) replace "svn" with "svn-history/r123".')) - # TODO(epoger): Rather than telling the user to run "svn ls" to get the list - # of builders, add a --list-builders option that will print the list. + '"%default".')) required_params.append('builder') + # TODO(epoger): Before https://codereview.chromium.org/309653005 , when this + # tool downloaded the JSON summaries from skia-autogen, it had the ability + # to get results as of a specific revision number. We should add similar + # functionality when retrieving the summaries from Google Storage. parser.add_option('--builder', action='store', type='string', help=('REQUIRED: Which builder to download results for. ' - 'To see a list of builders, run "svn ls %s".' % - DEFAULT_ACTUALS_BASE_URL)) + 'To see a list of builders, run with the ' + '--list-builders option set.')) required_params.append('dest_dir') parser.add_option('--dest-dir', action='store', type='string', @@ -180,8 +229,15 @@ def main(): default=DEFAULT_JSON_FILENAME, help=('JSON summary filename to read for each builder; ' 'defaults to "%default".')) + parser.add_option('--list-builders', action='store_true', + help=('List all available builders.')) (params, remaining_args) = parser.parse_args() + if params.list_builders: + dirs, _ = gcs_list_bucket_contents(bucket=GM_SUMMARIES_BUCKET) + print '\n'.join(dirs) + return + # Make sure all required options were set, # and that there were no items left over in the command line. for required_param in required_params: diff --git a/gm/rebaseline_server/server.py b/gm/rebaseline_server/server.py index 40874d603f..0680779670 100755 --- a/gm/rebaseline_server/server.py +++ b/gm/rebaseline_server/server.py @@ -250,6 +250,8 @@ class Server(object): PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_HTML_SUBDIR, "index.html"), config_pairs=config_pairs) + # TODO(epoger): Create shareable functions within download_actuals.py that + # we can use both there and here to download the actual image results. if actuals_repo_url: self._actuals_repo = _create_svn_checkout( dir_path=actuals_dir, repo_url=actuals_repo_url) |