#!/usr/bin/python """ Copyright 2014 Google Inc. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. Download actual GM results for a particular builder. """ # System-level imports import contextlib import optparse import os import posixpath import re import shutil import sys import urllib import urllib2 import urlparse # Imports from within Skia # # We need to add the 'gm' and 'tools' directories, so that we can import # gm_json.py and buildbot_globals.py. # # Make sure that these dirs are in the PYTHONPATH, but add them at the *end* # so any dirs that are already in the PYTHONPATH will be preferred. # # TODO(epoger): Is it OK for this to depend on the 'tools' dir, given that # the tools dir is dependent on the 'gm' dir (to import gm_json.py)? TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) GM_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'gm') TOOLS_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'tools') if GM_DIRECTORY not in sys.path: sys.path.append(GM_DIRECTORY) if TOOLS_DIRECTORY not in sys.path: sys.path.append(TOOLS_DIRECTORY) import buildbot_globals import gm_json DEFAULT_ACTUALS_BASE_URL = posixpath.join( buildbot_globals.Get('autogen_svn_url'), 'gm-actual') DEFAULT_JSON_FILENAME = 'actual-results.json' class Download(object): def __init__(self, actuals_base_url=DEFAULT_ACTUALS_BASE_URL, json_filename=DEFAULT_JSON_FILENAME, gm_actuals_root_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL): """ Args: actuals_base_url: URL pointing at the root directory containing all actual-results.json files, e.g., http://domain.name/path/to/dir OR file:///absolute/path/to/localdir json_filename: The JSON filename to read from within each directory. gm_actuals_root_url: Base URL under which the actually-generated-by-bots GM images are stored. """ self._actuals_base_url = actuals_base_url self._json_filename = json_filename self._gm_actuals_root_url = gm_actuals_root_url self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN) def fetch(self, builder_name, dest_dir): """ Downloads actual GM results for a particular builder. Args: builder_name: which builder to download results of dest_dir: path to directory where the image files will be written; if the directory does not exist yet, it will be created TODO(epoger): Display progress info. Right now, it can take a long time to download all of the results, and there is no indication of progress. TODO(epoger): Download multiple images in parallel to speed things up. """ json_url = posixpath.join(self._actuals_base_url, builder_name, self._json_filename) json_contents = urllib2.urlopen(json_url).read() results_dict = gm_json.LoadFromString(json_contents) actual_results_dict = results_dict[gm_json.JSONKEY_ACTUALRESULTS] for result_type in sorted(actual_results_dict.keys()): results_of_this_type = actual_results_dict[result_type] if not results_of_this_type: continue for image_name in sorted(results_of_this_type.keys()): (test, config) = self._image_filename_re.match(image_name).groups() (hash_type, hash_digest) = results_of_this_type[image_name] source_url = gm_json.CreateGmActualUrl( test_name=test, hash_type=hash_type, hash_digest=hash_digest, gm_actuals_root_url=self._gm_actuals_root_url) dest_path = os.path.join(dest_dir, config, test + '.png') copy_contents(source_url=source_url, dest_path=dest_path, create_subdirs_if_needed=True) def create_filepath_url(filepath): """ Returns a file:/// URL pointing at the given filepath on local disk. For now, this is only used by unittests, but I anticipate it being useful in production, as a way for developers to run rebaseline_server over locally generated images. TODO(epoger): Move this function, and copy_contents(), into a shared utility module. They are generally useful. Args: filepath: string; path to a file on local disk (may be absolute or relative, and the file does not need to exist) Returns: A file:/// URL pointing at the file. Regardless of whether filepath was specified as a relative or absolute path, the URL will contain an absolute path to the file. Raises: An Exception, if filepath is already a URL. """ if urlparse.urlparse(filepath).scheme: raise Exception('"%s" is already a URL' % filepath) return urlparse.urljoin( 'file:', urllib.pathname2url(os.path.abspath(filepath))) def copy_contents(source_url, dest_path, create_subdirs_if_needed=False): """ Copies the full contents of the URL 'source_url' into filepath 'dest_path'. Args: source_url: string; complete URL to read from dest_path: string; complete filepath to write to (may be absolute or relative) create_subdirs_if_needed: boolean; whether to create subdirectories as needed to create dest_path Raises: Some subclass of Exception if unable to read source_url or write dest_path. """ if create_subdirs_if_needed: dest_dir = os.path.dirname(dest_path) if not os.path.exists(dest_dir): os.makedirs(dest_dir) with contextlib.closing(urllib.urlopen(source_url)) as source_handle: with open(dest_path, 'wb') as dest_handle: shutil.copyfileobj(fsrc=source_handle, fdst=dest_handle) def main(): parser = optparse.OptionParser() required_params = [] parser.add_option('--actuals-base-url', action='store', type='string', default=DEFAULT_ACTUALS_BASE_URL, help=('Base URL from which to read files containing JSON ' 'summaries of actual GM results; defaults to ' '"%default". To get a specific revision (useful for ' 'trybots) replace "svn" with "svn-history/r123".')) # TODO(epoger): Rather than telling the user to run "svn ls" to get the list # of builders, add a --list-builders option that will print the list. required_params.append('builder') parser.add_option('--builder', action='store', type='string', help=('REQUIRED: Which builder to download results for. ' 'To see a list of builders, run "svn ls %s".' % DEFAULT_ACTUALS_BASE_URL)) required_params.append('dest_dir') parser.add_option('--dest-dir', action='store', type='string', help=('REQUIRED: Directory where all images should be ' 'written. If this directory does not exist yet, it ' 'will be created.')) parser.add_option('--json-filename', action='store', type='string', default=DEFAULT_JSON_FILENAME, help=('JSON summary filename to read for each builder; ' 'defaults to "%default".')) (params, remaining_args) = parser.parse_args() # Make sure all required options were set, # and that there were no items left over in the command line. for required_param in required_params: if not getattr(params, required_param): raise Exception('required option \'%s\' was not set' % required_param) if len(remaining_args) is not 0: raise Exception('extra items specified in the command line: %s' % remaining_args) downloader = Download(actuals_base_url=params.actuals_base_url) downloader.fetch(builder_name=params.builder, dest_dir=params.dest_dir) if __name__ == '__main__': main()