#!/usr/bin/python """ Copyright 2014 Google Inc. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. Download actual GM results for a particular builder. """ # System-level imports import httplib import logging import optparse import os import posixpath import re import urllib2 # Must fix up PYTHONPATH before importing from within Skia import rs_fixpypath # pylint: disable=W0611 # Imports from within Skia from py.utils import gs_utils from py.utils import url_utils import buildbot_globals import gm_json GM_SUMMARIES_BUCKET = buildbot_globals.Get('gm_summaries_bucket') DEFAULT_ACTUALS_BASE_URL = ( 'http://storage.googleapis.com/%s' % GM_SUMMARIES_BUCKET) DEFAULT_JSON_FILENAME = 'actual-results.json' class Download(object): def __init__(self, actuals_base_url=DEFAULT_ACTUALS_BASE_URL, json_filename=DEFAULT_JSON_FILENAME, gm_actuals_root_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL): """ Args: actuals_base_url: URL pointing at the root directory containing all actual-results.json files, e.g., http://domain.name/path/to/dir OR file:///absolute/path/to/localdir json_filename: The JSON filename to read from within each directory. gm_actuals_root_url: Base URL under which the actually-generated-by-bots GM images are stored. """ self._actuals_base_url = actuals_base_url self._json_filename = json_filename self._gm_actuals_root_url = gm_actuals_root_url self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN) def fetch(self, builder_name, dest_dir): """ Downloads actual GM results for a particular builder. Args: builder_name: which builder to download results of dest_dir: path to directory where the image files will be written; if the directory does not exist yet, it will be created TODO(epoger): Display progress info. Right now, it can take a long time to download all of the results, and there is no indication of progress. TODO(epoger): Download multiple images in parallel to speed things up. """ json_url = posixpath.join(self._actuals_base_url, builder_name, self._json_filename) json_contents = urllib2.urlopen(json_url).read() results_dict = gm_json.LoadFromString(json_contents) actual_results_dict = results_dict[gm_json.JSONKEY_ACTUALRESULTS] for result_type in sorted(actual_results_dict.keys()): results_of_this_type = actual_results_dict[result_type] if not results_of_this_type: continue for image_name in sorted(results_of_this_type.keys()): (test, config) = self._image_filename_re.match(image_name).groups() (hash_type, hash_digest) = results_of_this_type[image_name] source_url = gm_json.CreateGmActualUrl( test_name=test, hash_type=hash_type, hash_digest=hash_digest, gm_actuals_root_url=self._gm_actuals_root_url) dest_path = os.path.join(dest_dir, config, test + '.png') url_utils.copy_contents(source_url=source_url, dest_path=dest_path, create_subdirs_if_needed=True) def get_builders_list(summaries_bucket=GM_SUMMARIES_BUCKET): """ Returns the list of builders we have actual results for. Args: summaries_bucket: Google Cloud Storage bucket containing the summary JSON files """ dirs, _ = gs_utils.GSUtils().list_bucket_contents(bucket=GM_SUMMARIES_BUCKET) return dirs class ActualLocation(object): def __init__(self, bucket, path, generation): self.bucket = bucket self.path = path self.generation = generation class TipOfTreeActuals(object): def __init__(self, summaries_bucket=GM_SUMMARIES_BUCKET, json_filename=DEFAULT_JSON_FILENAME): """ Args: summaries_bucket: URL pointing at the root directory containing all actual-results.json files, e.g., http://domain.name/path/to/dir OR file:///absolute/path/to/localdir json_filename: The JSON filename to read from within each directory. """ self._json_filename = json_filename self._summaries_bucket = summaries_bucket def description(self): return 'gm_summaries_bucket %s' % (self._summaries_bucket,) def get_builders(self): """ Returns the list of builders we have actual results for. {builder:string -> ActualLocation} """ dirs = get_builders_list(self._summaries_bucket) result = dict() for builder in dirs: result[builder] = ActualLocation( self._summaries_bucket, "%s/%s" % (builder, self._json_filename), None) return result class RietveldIssueActuals(object): def __init__(self, issue, json_filename=DEFAULT_JSON_FILENAME): """ Args: issue: The rietveld issue from which to obtain actuals. json_filename: The JSON filename to read from within each directory. """ self._issue = issue self._json_filename = json_filename def description(self): return 'rietveld issue %s' % (self._issue,) def get_builders(self): """ Returns the actuals for the given rietveld issue's tryjobs. {builder:string -> ActualLocation} e.g. {'Test-Android-Xoom-Tegra2-Arm7-Release': ( 'chromium-skia-gm-summaries', 'Test-Android-Xoom-Tegra2-Arm7-Release-Trybot/actual-results.json', '1415041165535000')} """ result = dict() json_filename_re = re.compile( 'Created: gs://([^/]+)/((?:[^/]+/)+%s)#(\d+)' % re.escape(self._json_filename)) codereview_api_url = 'https://codereview.chromium.org/api' upload_gm_step_url = '/steps/Upload GM Results/logs/stdio' logging.info('Fetching issue %s ...' % (self._issue,)) json_issue_url = '%s/%s' % (codereview_api_url, self._issue) json_issue_data = urllib2.urlopen(json_issue_url).read() issue_dict = gm_json.LoadFromString(json_issue_data) patchsets = issue_dict.get("patchsets", []) patchset = patchsets[-1] if not patchset: logging.warning('No patchsets for rietveld issue %s.' % (self._issue,)) return result logging.info('Fetching issue %s patch %s...' % (self._issue, patchset)) json_patchset_url = '%s/%s/%s' % (codereview_api_url, self._issue, patchset) json_patchset_data = urllib2.urlopen(json_patchset_url).read() patchset_dict = gm_json.LoadFromString(json_patchset_data) # try_job_results is ordered reverse chronologically try_job_results = patchset_dict.get('try_job_results', []) for try_job_result in try_job_results: try_builder = try_job_result.get('builder', '') if not try_builder.endswith('-Trybot'): logging.warning('Builder %s is not a trybot?' % (try_builder,)) continue builder = try_builder[:-len('-Trybot')] if builder in result: continue logging.info('Fetching issue %s patch %s try %s...' % (self._issue, patchset, try_builder)) build_url = try_job_result.get('url', '') if build_url is None: logging.warning('Builder %s has not started.' % (try_builder,)) continue gm_upload_output_url = build_url + urllib2.quote(upload_gm_step_url) logging.info('Fetching %s ...' % (gm_upload_output_url,)) # Tryjobs might not produce the step, but don't let that fail everything. gm_upload_output = None try: gm_upload_output = urllib2.urlopen(gm_upload_output_url).read() except (urllib2.HTTPError, urllib2.URLError, httplib.HTTPException) as e: logging.warning(e) except Exception: logging.exception('Error opening %s .' % (gm_upload_output_url,)) if not gm_upload_output: logging.warning('Could not fetch %s .' % (gm_upload_output_url,)) continue json_filename_match = json_filename_re.search(gm_upload_output) if json_filename_match: logging.info('Found issue %s patch %s try %s result gs://%s/%s#%s .' % (self._issue, patchset, builder, json_filename_match.group(1), json_filename_match.group(2), json_filename_match.group(3))) result[builder] = ActualLocation(json_filename_match.group(1), json_filename_match.group(2), json_filename_match.group(3)) else: logging.warning('Did not find %s for issue %s patch %s try %s.' % (self._json_filename, self._issue, patchset, try_builder)) return result def main(): parser = optparse.OptionParser() required_params = [] parser.add_option('--actuals-base-url', action='store', type='string', default=DEFAULT_ACTUALS_BASE_URL, help=('Base URL from which to read files containing JSON ' 'summaries of actual GM results; defaults to ' '"%default".')) required_params.append('builder') # TODO(epoger): Before https://codereview.chromium.org/309653005 , when this # tool downloaded the JSON summaries from skia-autogen, it had the ability # to get results as of a specific revision number. We should add similar # functionality when retrieving the summaries from Google Storage. parser.add_option('--builder', action='store', type='string', help=('REQUIRED: Which builder to download results for. ' 'To see a list of builders, run with the ' '--list-builders option set.')) required_params.append('dest_dir') parser.add_option('--dest-dir', action='store', type='string', help=('REQUIRED: Directory where all images should be ' 'written. If this directory does not exist yet, it ' 'will be created.')) parser.add_option('--json-filename', action='store', type='string', default=DEFAULT_JSON_FILENAME, help=('JSON summary filename to read for each builder; ' 'defaults to "%default".')) parser.add_option('--list-builders', action='store_true', help=('List all available builders.')) (params, remaining_args) = parser.parse_args() if params.list_builders: print '\n'.join(get_builders_list()) return # Make sure all required options were set, # and that there were no items left over in the command line. for required_param in required_params: if not getattr(params, required_param): raise Exception('required option \'%s\' was not set' % required_param) if len(remaining_args) is not 0: raise Exception('extra items specified in the command line: %s' % remaining_args) downloader = Download(actuals_base_url=params.actuals_base_url) downloader.fetch(builder_name=params.builder, dest_dir=params.dest_dir) if __name__ == '__main__': main()