diff options
| -rwxr-xr-x | gm/rebaseline_server/compare_configs.py | 22 | ||||
| -rwxr-xr-x | gm/rebaseline_server/compare_rendered_pictures.py | 21 | ||||
| -rwxr-xr-x | gm/rebaseline_server/compare_to_expectations.py | 21 | ||||
| -rwxr-xr-x | gm/rebaseline_server/download_actuals.py | 129 | ||||
| -rwxr-xr-x | gm/rebaseline_server/download_actuals_test.py | 44 | ||||
| -rwxr-xr-x | gm/rebaseline_server/fix_pythonpath.py | 21 | ||||
| -rwxr-xr-x | gm/rebaseline_server/results.py | 13 | ||||
| -rwxr-xr-x | gm/rebaseline_server/server.py | 156 | ||||
| -rw-r--r-- | tools/pyutils/__init__.py | 0 | ||||
| -rwxr-xr-x | tools/pyutils/gs_utils.py | 82 | ||||
| -rwxr-xr-x | tools/pyutils/url_utils.py | 63 | ||||
| -rwxr-xr-x | tools/pyutils/url_utils_test.py | 61 | ||||
| -rwxr-xr-x | tools/test_all.py | 32 | ||||
| -rw-r--r-- | tools/tests/__init__.py | 0 | ||||
| -rwxr-xr-x | tools/tests/run.sh | 2 | ||||
| -rwxr-xr-x | tools/tests/run_all.py | 22 |
16 files changed, 318 insertions, 371 deletions
diff --git a/gm/rebaseline_server/compare_configs.py b/gm/rebaseline_server/compare_configs.py index 4075da4263..aa26ba6f37 100755 --- a/gm/rebaseline_server/compare_configs.py +++ b/gm/rebaseline_server/compare_configs.py @@ -14,12 +14,28 @@ import argparse import fnmatch import json import logging +import os import re +import sys import time # Imports from within Skia -import fix_pythonpath # must do this first -from pyutils import url_utils +# +# TODO(epoger): Once we move the create_filepath_url() function out of +# download_actuals into a shared utility module, we won't need to import +# download_actuals anymore. +# +# We need to add the 'gm' directory, so that we can import gm_json.py within +# that directory. That script allows us to parse the actual-results.json file +# written out by the GM tool. +# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* +# so any dirs that are already in the PYTHONPATH will be preferred. +PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) +GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY) +TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY) +if GM_DIRECTORY not in sys.path: + sys.path.append(GM_DIRECTORY) +import download_actuals import gm_json import imagediffdb import imagepair @@ -55,7 +71,7 @@ class ConfigComparisons(results.BaseComparisons): self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) self._diff_base_url = ( diff_base_url or - url_utils.create_filepath_url(generated_images_root)) + download_actuals.create_filepath_url(generated_images_root)) self._actuals_root = actuals_root self._load_config_pairs(configs) self._timestamp = int(time.time()) diff --git a/gm/rebaseline_server/compare_rendered_pictures.py b/gm/rebaseline_server/compare_rendered_pictures.py index 73d0627ba0..75a80d4de2 100755 --- a/gm/rebaseline_server/compare_rendered_pictures.py +++ b/gm/rebaseline_server/compare_rendered_pictures.py @@ -13,11 +13,26 @@ Compare results of two render_pictures runs. import logging import os import re +import sys import time # Imports from within Skia -import fix_pythonpath # must do this first -from pyutils import url_utils +# +# TODO(epoger): Once we move the create_filepath_url() function out of +# download_actuals into a shared utility module, we won't need to import +# download_actuals anymore. +# +# We need to add the 'gm' directory, so that we can import gm_json.py within +# that directory. That script allows us to parse the actual-results.json file +# written out by the GM tool. +# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* +# so any dirs that are already in the PYTHONPATH will be preferred. +PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) +GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY) +TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY) +if GM_DIRECTORY not in sys.path: + sys.path.append(GM_DIRECTORY) +import download_actuals import gm_json import imagediffdb import imagepair @@ -59,7 +74,7 @@ class RenderedPicturesComparisons(results.BaseComparisons): self._image_base_url = image_base_url self._diff_base_url = ( diff_base_url or - url_utils.create_filepath_url(generated_images_root)) + download_actuals.create_filepath_url(generated_images_root)) self._load_result_pairs(actuals_root, subdirs) self._timestamp = int(time.time()) logging.info('Results complete; took %d seconds.' % diff --git a/gm/rebaseline_server/compare_to_expectations.py b/gm/rebaseline_server/compare_to_expectations.py index 1a93c6674f..e9677bdcb8 100755 --- a/gm/rebaseline_server/compare_to_expectations.py +++ b/gm/rebaseline_server/compare_to_expectations.py @@ -20,8 +20,22 @@ import sys import time # Imports from within Skia -import fix_pythonpath # must do this first -from pyutils import url_utils +# +# TODO(epoger): Once we move the create_filepath_url() function out of +# download_actuals into a shared utility module, we won't need to import +# download_actuals anymore. +# +# We need to add the 'gm' directory, so that we can import gm_json.py within +# that directory. That script allows us to parse the actual-results.json file +# written out by the GM tool. +# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* +# so any dirs that are already in the PYTHONPATH will be preferred. +PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) +GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY) +TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY) +if GM_DIRECTORY not in sys.path: + sys.path.append(GM_DIRECTORY) +import download_actuals import gm_json import imagediffdb import imagepair @@ -33,7 +47,6 @@ EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [ results.KEY__EXPECTATIONS__IGNOREFAILURE, results.KEY__EXPECTATIONS__REVIEWED, ] -TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm') DEFAULT_IGNORE_FAILURES_FILE = 'ignored-tests.txt' @@ -75,7 +88,7 @@ class ExpectationComparisons(results.BaseComparisons): self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) self._diff_base_url = ( diff_base_url or - url_utils.create_filepath_url(generated_images_root)) + download_actuals.create_filepath_url(generated_images_root)) self._actuals_root = actuals_root self._expected_root = expected_root self._ignore_failures_on_these_tests = [] diff --git a/gm/rebaseline_server/download_actuals.py b/gm/rebaseline_server/download_actuals.py index 2f92898fd4..636958be6b 100755 --- a/gm/rebaseline_server/download_actuals.py +++ b/gm/rebaseline_server/download_actuals.py @@ -10,19 +10,44 @@ Download actual GM results for a particular builder. """ # System-level imports +import contextlib import optparse import os import posixpath import re +import shutil +import sys +import urllib import urllib2 +import urlparse # Imports from within Skia -import fix_pythonpath # must do this first -from pyutils import gs_utils -from pyutils import url_utils +# +# We need to add the 'gm' and 'tools' directories, so that we can import +# gm_json.py and buildbot_globals.py. +# +# Make sure that these dirs are in the PYTHONPATH, but add them at the *end* +# so any dirs that are already in the PYTHONPATH will be preferred. +# +# TODO(epoger): Is it OK for this to depend on the 'tools' dir, given that +# the tools dir is dependent on the 'gm' dir (to import gm_json.py)? +TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) +GM_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'gm') +TOOLS_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'tools') +if GM_DIRECTORY not in sys.path: + sys.path.append(GM_DIRECTORY) +if TOOLS_DIRECTORY not in sys.path: + sys.path.append(TOOLS_DIRECTORY) import buildbot_globals import gm_json +# Imports from third-party code +APICLIENT_DIRECTORY = os.path.join( + TRUNK_DIRECTORY, 'third_party', 'externals', 'google-api-python-client') +if APICLIENT_DIRECTORY not in sys.path: + sys.path.append(APICLIENT_DIRECTORY) +from googleapiclient.discovery import build as build_service + GM_SUMMARIES_BUCKET = buildbot_globals.Get('gm_summaries_bucket') DEFAULT_ACTUALS_BASE_URL = ( @@ -80,19 +105,98 @@ class Download(object): test_name=test, hash_type=hash_type, hash_digest=hash_digest, gm_actuals_root_url=self._gm_actuals_root_url) dest_path = os.path.join(dest_dir, config, test + '.png') - url_utils.copy_contents(source_url=source_url, dest_path=dest_path, - create_subdirs_if_needed=True) + # TODO(epoger): To speed this up, we should only download files that + # we don't already have on local disk. + copy_contents(source_url=source_url, dest_path=dest_path, + create_subdirs_if_needed=True) + + +def create_filepath_url(filepath): + """ Returns a file:/// URL pointing at the given filepath on local disk. + For now, this is only used by unittests, but I anticipate it being useful + in production, as a way for developers to run rebaseline_server over locally + generated images. -def get_builders_list(summaries_bucket=GM_SUMMARIES_BUCKET): - """ Returns the list of builders we have actual results for. + TODO(epoger): Move this function, and copy_contents(), into a shared + utility module. They are generally useful. Args: - summaries_bucket: Google Cloud Storage bucket containing the summary - JSON files + filepath: string; path to a file on local disk (may be absolute or relative, + and the file does not need to exist) + + Returns: + A file:/// URL pointing at the file. Regardless of whether filepath was + specified as a relative or absolute path, the URL will contain an + absolute path to the file. + + Raises: + An Exception, if filepath is already a URL. """ - dirs, _ = gs_utils.list_bucket_contents(bucket=GM_SUMMARIES_BUCKET) - return dirs + if urlparse.urlparse(filepath).scheme: + raise Exception('"%s" is already a URL' % filepath) + return urlparse.urljoin( + 'file:', urllib.pathname2url(os.path.abspath(filepath))) + + +def copy_contents(source_url, dest_path, create_subdirs_if_needed=False): + """ Copies the full contents of the URL 'source_url' into + filepath 'dest_path'. + + Args: + source_url: string; complete URL to read from + dest_path: string; complete filepath to write to (may be absolute or + relative) + create_subdirs_if_needed: boolean; whether to create subdirectories as + needed to create dest_path + + Raises: + Some subclass of Exception if unable to read source_url or write dest_path. + """ + if create_subdirs_if_needed: + dest_dir = os.path.dirname(dest_path) + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + with contextlib.closing(urllib.urlopen(source_url)) as source_handle: + with open(dest_path, 'wb') as dest_handle: + shutil.copyfileobj(fsrc=source_handle, fdst=dest_handle) + + +def gcs_list_bucket_contents(bucket, subdir=None): + """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple. + + Uses the API documented at + https://developers.google.com/storage/docs/json_api/v1/objects/list + + Args: + bucket: name of the Google Storage bucket + subdir: directory within the bucket to list, or None for root directory + """ + # The GCS command relies on the subdir name (if any) ending with a slash. + if subdir and not subdir.endswith('/'): + subdir += '/' + subdir_length = len(subdir) if subdir else 0 + + storage = build_service('storage', 'v1') + command = storage.objects().list( + bucket=bucket, delimiter='/', fields='items(name),prefixes', + prefix=subdir) + results = command.execute() + + # The GCS command returned two subdicts: + # prefixes: the full path of every directory within subdir, with trailing '/' + # items: property dict for each file object within subdir + # (including 'name', which is full path of the object) + dirs = [] + for dir_fullpath in results.get('prefixes', []): + dir_basename = dir_fullpath[subdir_length:] + dirs.append(dir_basename[:-1]) # strip trailing slash + files = [] + for file_properties in results.get('items', []): + file_fullpath = file_properties['name'] + file_basename = file_fullpath[subdir_length:] + files.append(file_basename) + return (dirs, files) def main(): @@ -130,7 +234,8 @@ def main(): (params, remaining_args) = parser.parse_args() if params.list_builders: - print '\n'.join(get_builders_list()) + dirs, _ = gcs_list_bucket_contents(bucket=GM_SUMMARIES_BUCKET) + print '\n'.join(dirs) return # Make sure all required options were set, diff --git a/gm/rebaseline_server/download_actuals_test.py b/gm/rebaseline_server/download_actuals_test.py index c405a3cf57..8813530519 100755 --- a/gm/rebaseline_server/download_actuals_test.py +++ b/gm/rebaseline_server/download_actuals_test.py @@ -25,8 +25,6 @@ import tempfile import urllib # Imports from within Skia -import fix_pythonpath # must do this first -from pyutils import url_utils import base_unittest import download_actuals @@ -36,14 +34,52 @@ class DownloadTest(base_unittest.TestCase): def test_fetch(self): """Tests fetch() of GM results from actual-results.json .""" downloader = download_actuals.Download( - actuals_base_url=url_utils.create_filepath_url( + actuals_base_url=download_actuals.create_filepath_url( os.path.join(self._input_dir, 'gm-actuals')), - gm_actuals_root_url=url_utils.create_filepath_url( + gm_actuals_root_url=download_actuals.create_filepath_url( os.path.join(self._input_dir, 'fake-gm-imagefiles'))) downloader.fetch( builder_name='Test-Android-GalaxyNexus-SGX540-Arm7-Release', dest_dir=self._output_dir_actual) + def test_create_filepath_url(self): + """Tests create_filepath_url(). """ + with self.assertRaises(Exception): + url_or_path.create_filepath_url('http://1.2.3.4/path') + # Pass absolute filepath. + self.assertEquals( + download_actuals.create_filepath_url( + '%sdir%sfile' % (os.path.sep, os.path.sep)), + 'file:///dir/file') + # Pass relative filepath. + self.assertEquals( + download_actuals.create_filepath_url(os.path.join('dir', 'file')), + 'file://%s/dir/file' % urllib.pathname2url(os.getcwd())) + + def test_copy_contents(self): + """Tests copy_contents(). """ + contents = 'these are the contents' + tempdir_path = tempfile.mkdtemp() + try: + source_path = os.path.join(tempdir_path, 'source') + source_url = download_actuals.create_filepath_url(source_path) + with open(source_path, 'w') as source_handle: + source_handle.write(contents) + dest_path = os.path.join(tempdir_path, 'new_subdir', 'dest') + # Destination subdir does not exist, so copy_contents() should fail + # if create_subdirs_if_needed is False. + with self.assertRaises(Exception): + download_actuals.copy_contents(source_url=source_url, + dest_path=dest_path, + create_subdirs_if_needed=False) + # If create_subdirs_if_needed is True, it should work. + download_actuals.copy_contents(source_url=source_url, + dest_path=dest_path, + create_subdirs_if_needed=True) + self.assertEquals(open(dest_path).read(), contents) + finally: + shutil.rmtree(tempdir_path) + def main(): base_unittest.main(DownloadTest) diff --git a/gm/rebaseline_server/fix_pythonpath.py b/gm/rebaseline_server/fix_pythonpath.py deleted file mode 100755 index ed578ced10..0000000000 --- a/gm/rebaseline_server/fix_pythonpath.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/python - -""" -Copyright 2014 Google Inc. - -Use of this source code is governed by a BSD-style license that can be -found in the LICENSE file. - -Adds [trunk]/gm and [trunk]/tools to PYTHONPATH, if they aren't already there. -""" - -import os -import sys - -TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) -GM_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'gm') -TOOLS_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'tools') -if GM_DIRECTORY not in sys.path: - sys.path.append(GM_DIRECTORY) -if TOOLS_DIRECTORY not in sys.path: - sys.path.append(TOOLS_DIRECTORY) diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/results.py index d17bc3d184..70b23425df 100755 --- a/gm/rebaseline_server/results.py +++ b/gm/rebaseline_server/results.py @@ -13,9 +13,19 @@ Repackage expected/actual GM results as needed by our HTML rebaseline viewer. import fnmatch import os import re +import sys # Imports from within Skia -import fix_pythonpath # must do this first +# +# We need to add the 'gm' directory, so that we can import gm_json.py within +# that directory. That script allows us to parse the actual-results.json file +# written out by the GM tool. +# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* +# so any dirs that are already in the PYTHONPATH will be preferred. +PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) +GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY) +if GM_DIRECTORY not in sys.path: + sys.path.append(GM_DIRECTORY) import gm_json import imagepairset @@ -47,7 +57,6 @@ KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config) -PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) DEFAULT_ACTUALS_DIR = '.gm-actuals' DEFAULT_GENERATED_IMAGES_ROOT = os.path.join( PARENT_DIRECTORY, '.generated-images') diff --git a/gm/rebaseline_server/server.py b/gm/rebaseline_server/server.py index 0079ec55ff..0680779670 100755 --- a/gm/rebaseline_server/server.py +++ b/gm/rebaseline_server/server.py @@ -20,14 +20,28 @@ import re import shutil import socket import subprocess +import sys import thread import threading import time import urlparse # Imports from within Skia -import fix_pythonpath # must do this first -from pyutils import gs_utils +# +# We need to add the 'tools' directory for svn.py, and the 'gm' directory for +# gm_json.py . +# that directory. +# Make sure that the 'tools' dir is in the PYTHONPATH, but add it at the *end* +# so any dirs that are already in the PYTHONPATH will be preferred. +PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) +GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY) +TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY) +TOOLS_DIRECTORY = os.path.join(TRUNK_DIRECTORY, 'tools') +if TOOLS_DIRECTORY not in sys.path: + sys.path.append(TOOLS_DIRECTORY) +import svn +if GM_DIRECTORY not in sys.path: + sys.path.append(GM_DIRECTORY) import gm_json # Imports from local dir @@ -37,7 +51,6 @@ import gm_json # https://codereview.chromium.org/195943004/diff/1/gm/rebaseline_server/server.py#newcode44 import compare_configs import compare_to_expectations -import download_actuals import imagepairset import results as results_mod @@ -61,12 +74,10 @@ KEY__EDITS__OLD_RESULTS_HASH = 'oldResultsHash' KEY__EDITS__OLD_RESULTS_TYPE = 'oldResultsType' DEFAULT_ACTUALS_DIR = results_mod.DEFAULT_ACTUALS_DIR -DEFAULT_GM_SUMMARIES_BUCKET = download_actuals.GM_SUMMARIES_BUCKET -DEFAULT_JSON_FILENAME = download_actuals.DEFAULT_JSON_FILENAME +DEFAULT_ACTUALS_REPO_REVISION = 'HEAD' +DEFAULT_ACTUALS_REPO_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual' DEFAULT_PORT = 8888 -PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) -TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(PARENT_DIRECTORY)) # Directory, relative to PARENT_DIRECTORY, within which the server will serve # out live results (not static files). RESULTS_SUBDIR = 'results' @@ -128,6 +139,24 @@ def _get_routable_ip_address(): return host +def _create_svn_checkout(dir_path, repo_url): + """Creates local checkout of an SVN repository at the specified directory + path, returning an svn.Svn object referring to the local checkout. + + Args: + dir_path: path to the local checkout; if this directory does not yet exist, + it will be created and the repo will be checked out into it + repo_url: URL of SVN repo to check out into dir_path (unless the local + checkout already exists) + Returns: an svn.Svn object referring to the local checkout. + """ + local_checkout = svn.Svn(dir_path) + if not os.path.isdir(dir_path): + os.makedirs(dir_path) + local_checkout.Checkout(repo_url, '.') + return local_checkout + + def _create_index(file_path, config_pairs): """Creates an index file linking to all results available from this server. @@ -184,18 +213,18 @@ class Server(object): def __init__(self, actuals_dir=DEFAULT_ACTUALS_DIR, - json_filename=DEFAULT_JSON_FILENAME, - gm_summaries_bucket=DEFAULT_GM_SUMMARIES_BUCKET, + actuals_repo_revision=DEFAULT_ACTUALS_REPO_REVISION, + actuals_repo_url=DEFAULT_ACTUALS_REPO_URL, port=DEFAULT_PORT, export=False, editable=True, reload_seconds=0, config_pairs=None, builder_regex_list=None): """ Args: actuals_dir: directory under which we will check out the latest actual GM results - json_filename: basename of the JSON summary file to load for each builder - gm_summaries_bucket: Google Storage bucket to download json_filename - files from; if None or '', don't fetch new actual-results files - at all, just compare to whatever files are already in actuals_dir + actuals_repo_revision: revision of actual-results.json files to process + actuals_repo_url: SVN repo to download actual-results.json files from; + if None or '', don't fetch new actual-results files at all, + just compare to whatever files are already in actuals_dir port: which TCP port to listen on for HTTP requests export: whether to allow HTTP clients on other hosts to access this server editable: whether HTTP clients are allowed to submit new baselines @@ -208,8 +237,8 @@ class Server(object): we will process. If None, process all builders. """ self._actuals_dir = actuals_dir - self._json_filename = json_filename - self._gm_summaries_bucket = gm_summaries_bucket + self._actuals_repo_revision = actuals_repo_revision + self._actuals_repo_url = actuals_repo_url self._port = port self._export = export self._editable = editable @@ -221,6 +250,11 @@ class Server(object): PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_HTML_SUBDIR, "index.html"), config_pairs=config_pairs) + # TODO(epoger): Create shareable functions within download_actuals.py that + # we can use both there and here to download the actual image results. + if actuals_repo_url: + self._actuals_repo = _create_svn_checkout( + dir_path=actuals_dir, repo_url=actuals_repo_url) # Reentrant lock that must be held whenever updating EITHER of: # 1. self._results @@ -268,66 +302,26 @@ class Server(object): with self.results_rlock: if invalidate: self._results = None - if self._gm_summaries_bucket: + if self._actuals_repo_url: logging.info( - 'Updating GM result summaries in %s from gm_summaries_bucket %s ...' - % (self._actuals_dir, self._gm_summaries_bucket)) - - # Clean out actuals_dir first, in case some builders have gone away - # since we last ran. - if os.path.isdir(self._actuals_dir): - shutil.rmtree(self._actuals_dir) - - # Get the list of builders we care about. - all_builders = download_actuals.get_builders_list( - summaries_bucket=self._gm_summaries_bucket) - if self._builder_regex_list: - matching_builders = [] - for builder in all_builders: - for regex in self._builder_regex_list: - if re.match(regex, builder): - matching_builders.append(builder) - break # go on to the next builder, no need to try more regexes - else: - matching_builders = all_builders - - # Download the JSON file for each builder we care about. - # - # TODO(epoger): When this is a large number of builders, we would be - # better off downloading them in parallel! - for builder in matching_builders: - gs_utils.download_file( - source_bucket=self._gm_summaries_bucket, - source_path=posixpath.join(builder, self._json_filename), - dest_path=os.path.join(self._actuals_dir, builder, - self._json_filename), - create_subdirs_if_needed=True) + 'Updating actual GM results in %s to revision %s from repo %s ...' + % ( + self._actuals_dir, self._actuals_repo_revision, + self._actuals_repo_url)) + self._actuals_repo.Update( + path='.', revision=self._actuals_repo_revision) # We only update the expectations dir if the server was run with a # nonzero --reload argument; otherwise, we expect the user to maintain # her own expectations as she sees fit. # - # Because the Skia repo is hosted using git, and git does not + # Because the Skia repo is moving from SVN to git, and git does not # support updating a single directory tree, we have to update the entire # repo checkout. # # Because Skia uses depot_tools, we have to update using "gclient sync" - # instead of raw git commands. - # - # TODO(epoger): Fetch latest expectations in some other way. - # Eric points out that our official documentation recommends an - # unmanaged Skia checkout, so "gclient sync" will not bring down updated - # expectations from origin/master-- you'd have to do a "git pull" of - # some sort instead. - # However, the live rebaseline_server at - # http://skia-tree-status.appspot.com/redirect/rebaseline-server (which - # is probably the only user of the --reload flag!) uses a managed - # checkout, so "gclient sync" works in that case. - # Probably the best idea is to avoid all of this nonsense by fetching - # updated expectations into a temp directory, and leaving the rest of - # the checkout alone. This could be done using "git show", or by - # downloading individual expectation JSON files from - # skia.googlesource.com . + # instead of raw git (or SVN) update. Happily, this will work whether + # the checkout was created using git or SVN. if self._reload_seconds: logging.info( 'Updating expected GM results in %s by syncing Skia repo ...' % @@ -629,11 +623,18 @@ def main(): 'actual GM results. If this directory does not ' 'exist, it will be created. Defaults to %(default)s'), default=DEFAULT_ACTUALS_DIR) - # TODO(epoger): Before https://codereview.chromium.org/310093003 , - # when this tool downloaded the JSON summaries from skia-autogen, - # it had an --actuals-revision the caller could specify to download - # actual results as of a specific point in time. We should add similar - # functionality when retrieving the summaries from Google Storage. + parser.add_argument('--actuals-repo', + help=('URL of SVN repo to download actual-results.json ' + 'files from. Defaults to %(default)s ; if set to ' + 'empty string, just compare to actual-results ' + 'already found in ACTUALS_DIR.'), + default=DEFAULT_ACTUALS_REPO_URL) + parser.add_argument('--actuals-revision', + help=('revision of actual-results.json files to process. ' + 'Defaults to %(default)s . Beware of setting this ' + 'argument in conjunction with --editable; you ' + 'probably only want to edit results at HEAD.'), + default=DEFAULT_ACTUALS_REPO_REVISION) parser.add_argument('--builders', metavar='BUILDER_REGEX', nargs='+', help=('Only process builders matching these regular ' 'expressions. If unspecified, process all ' @@ -651,17 +652,6 @@ def main(): 'to access this server. WARNING: doing so will ' 'allow users on other hosts to modify your ' 'GM expectations, if combined with --editable.')) - parser.add_argument('--gm-summaries-bucket', - help=('Google Cloud Storage bucket to download ' - 'JSON_FILENAME files from. ' - 'Defaults to %(default)s ; if set to ' - 'empty string, just compare to actual-results ' - 'already found in ACTUALS_DIR.'), - default=DEFAULT_GM_SUMMARIES_BUCKET) - parser.add_argument('--json-filename', - help=('JSON summary filename to read for each builder; ' - 'defaults to %(default)s.'), - default=DEFAULT_JSON_FILENAME) parser.add_argument('--port', type=int, help=('Which TCP port to listen on for HTTP requests; ' 'defaults to %(default)s'), @@ -682,8 +672,8 @@ def main(): global _SERVER _SERVER = Server(actuals_dir=args.actuals_dir, - json_filename=args.json_filename, - gm_summaries_bucket=args.gm_summaries_bucket, + actuals_repo_revision=args.actuals_revision, + actuals_repo_url=args.actuals_repo, port=args.port, export=args.export, editable=args.editable, reload_seconds=args.reload, config_pairs=config_pairs, builder_regex_list=args.builders) diff --git a/tools/pyutils/__init__.py b/tools/pyutils/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 --- a/tools/pyutils/__init__.py +++ /dev/null diff --git a/tools/pyutils/gs_utils.py b/tools/pyutils/gs_utils.py deleted file mode 100755 index 745276e693..0000000000 --- a/tools/pyutils/gs_utils.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/python - -""" -Copyright 2014 Google Inc. - -Use of this source code is governed by a BSD-style license that can be -found in the LICENSE file. - -Utilities for accessing Google Cloud Storage. - -TODO(epoger): move this into tools/utils for broader use? -""" - -# System-level imports -import os -import posixpath -import sys - -# Imports from third-party code -TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) -APICLIENT_DIRECTORY = os.path.join( - TRUNK_DIRECTORY, 'third_party', 'externals', 'google-api-python-client') -if APICLIENT_DIRECTORY not in sys.path: - sys.path.append(APICLIENT_DIRECTORY) -from googleapiclient.discovery import build as build_service - -# Local imports -import url_utils - - -def download_file(source_bucket, source_path, dest_path, - create_subdirs_if_needed=False): - """ Downloads a single file from Google Cloud Storage to local disk. - - Args: - source_bucket: GCS bucket to download the file from - source_path: full path (Posix-style) within that bucket - dest_path: full path (local-OS-style) on local disk to copy the file to - create_subdirs_if_needed: boolean; whether to create subdirectories as - needed to create dest_path - """ - source_http_url = posixpath.join( - 'http://storage.googleapis.com', source_bucket, source_path) - url_utils.copy_contents(source_url=source_http_url, dest_path=dest_path, - create_subdirs_if_needed=create_subdirs_if_needed) - - -def list_bucket_contents(bucket, subdir=None): - """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple. - - Uses the API documented at - https://developers.google.com/storage/docs/json_api/v1/objects/list - - Args: - bucket: name of the Google Storage bucket - subdir: directory within the bucket to list, or None for root directory - """ - # The GCS command relies on the subdir name (if any) ending with a slash. - if subdir and not subdir.endswith('/'): - subdir += '/' - subdir_length = len(subdir) if subdir else 0 - - storage = build_service('storage', 'v1') - command = storage.objects().list( - bucket=bucket, delimiter='/', fields='items(name),prefixes', - prefix=subdir) - results = command.execute() - - # The GCS command returned two subdicts: - # prefixes: the full path of every directory within subdir, with trailing '/' - # items: property dict for each file object within subdir - # (including 'name', which is full path of the object) - dirs = [] - for dir_fullpath in results.get('prefixes', []): - dir_basename = dir_fullpath[subdir_length:] - dirs.append(dir_basename[:-1]) # strip trailing slash - files = [] - for file_properties in results.get('items', []): - file_fullpath = file_properties['name'] - file_basename = file_fullpath[subdir_length:] - files.append(file_basename) - return (dirs, files) diff --git a/tools/pyutils/url_utils.py b/tools/pyutils/url_utils.py deleted file mode 100755 index b107f560db..0000000000 --- a/tools/pyutils/url_utils.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/python - -""" -Copyright 2014 Google Inc. - -Use of this source code is governed by a BSD-style license that can be -found in the LICENSE file. - -Utilities for working with URLs. - -TODO(epoger): move this into tools/utils for broader use? -""" - -# System-level imports -import contextlib -import os -import shutil -import urllib -import urlparse - - -def create_filepath_url(filepath): - """ Returns a file:/// URL pointing at the given filepath on local disk. - - Args: - filepath: string; path to a file on local disk (may be absolute or relative, - and the file does not need to exist) - - Returns: - A file:/// URL pointing at the file. Regardless of whether filepath was - specified as a relative or absolute path, the URL will contain an - absolute path to the file. - - Raises: - An Exception, if filepath is already a URL. - """ - if urlparse.urlparse(filepath).scheme: - raise Exception('"%s" is already a URL' % filepath) - return urlparse.urljoin( - 'file:', urllib.pathname2url(os.path.abspath(filepath))) - - -def copy_contents(source_url, dest_path, create_subdirs_if_needed=False): - """ Copies the full contents of the URL 'source_url' into - filepath 'dest_path'. - - Args: - source_url: string; complete URL to read from - dest_path: string; complete filepath to write to (may be absolute or - relative) - create_subdirs_if_needed: boolean; whether to create subdirectories as - needed to create dest_path - - Raises: - Some subclass of Exception if unable to read source_url or write dest_path. - """ - if create_subdirs_if_needed: - dest_dir = os.path.dirname(dest_path) - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) - with contextlib.closing(urllib.urlopen(source_url)) as source_handle: - with open(dest_path, 'wb') as dest_handle: - shutil.copyfileobj(fsrc=source_handle, fdst=dest_handle) diff --git a/tools/pyutils/url_utils_test.py b/tools/pyutils/url_utils_test.py deleted file mode 100755 index ef3d8c8aaa..0000000000 --- a/tools/pyutils/url_utils_test.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/python - -""" -Copyright 2014 Google Inc. - -Use of this source code is governed by a BSD-style license that can be -found in the LICENSE file. - -Test url_utils.py -""" - -# System-level imports -import os -import shutil -import tempfile -import unittest -import urllib - -# Imports from within Skia -import url_utils - - -class UrlUtilsTest(unittest.TestCase): - - def test_create_filepath_url(self): - """Tests create_filepath_url(). """ - with self.assertRaises(Exception): - url_utils.create_filepath_url('http://1.2.3.4/path') - # Pass absolute filepath. - self.assertEquals( - url_utils.create_filepath_url( - '%sdir%sfile' % (os.path.sep, os.path.sep)), - 'file:///dir/file') - # Pass relative filepath. - self.assertEquals( - url_utils.create_filepath_url(os.path.join('dir', 'file')), - 'file://%s/dir/file' % urllib.pathname2url(os.getcwd())) - - def test_copy_contents(self): - """Tests copy_contents(). """ - contents = 'these are the contents' - tempdir_path = tempfile.mkdtemp() - try: - source_path = os.path.join(tempdir_path, 'source') - source_url = url_utils.create_filepath_url(source_path) - with open(source_path, 'w') as source_handle: - source_handle.write(contents) - dest_path = os.path.join(tempdir_path, 'new_subdir', 'dest') - # Destination subdir does not exist, so copy_contents() should fail - # if create_subdirs_if_needed is False. - with self.assertRaises(Exception): - url_utils.copy_contents(source_url=source_url, - dest_path=dest_path, - create_subdirs_if_needed=False) - # If create_subdirs_if_needed is True, it should work. - url_utils.copy_contents(source_url=source_url, - dest_path=dest_path, - create_subdirs_if_needed=True) - self.assertEquals(open(dest_path).read(), contents) - finally: - shutil.rmtree(tempdir_path) diff --git a/tools/test_all.py b/tools/test_all.py deleted file mode 100755 index 6467a2160b..0000000000 --- a/tools/test_all.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python - -""" -Copyright 2014 Google Inc. - -Use of this source code is governed by a BSD-style license that can be -found in the LICENSE file. - -Run all unittests within this directory tree, recursing into subdirectories. -""" - -import os -import unittest - -from tests import skimage_self_test - - -def main(): - # First, run any tests that cannot be automatically discovered (because - # they don't use Python's unittest framework). - skimage_self_test.main() - - # Now discover/run all tests that use Python's unittest framework. - suite = unittest.TestLoader().discover(os.path.dirname(__file__), - pattern='*_test.py') - results = unittest.TextTestRunner(verbosity=2).run(suite) - print repr(results) - if not results.wasSuccessful(): - raise Exception('failed one or more unittests') - -if __name__ == '__main__': - main() diff --git a/tools/tests/__init__.py b/tools/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 --- a/tools/tests/__init__.py +++ /dev/null diff --git a/tools/tests/run.sh b/tools/tests/run.sh index fd12711f49..6fca3cb6a2 100755 --- a/tools/tests/run.sh +++ b/tools/tests/run.sh @@ -201,7 +201,7 @@ jsondiff_test "$JSONDIFF_INPUT/old.json $JSONDIFF_INPUT/new.json" "$JSONDIFF_OUT # ('make tools/tests/run.sh work cross-platform') # -COMMAND="python tools/test_all.py" +COMMAND="python tools/tests/run_all.py" echo "$COMMAND" $COMMAND ret=$? diff --git a/tools/tests/run_all.py b/tools/tests/run_all.py new file mode 100755 index 0000000000..84886f758f --- /dev/null +++ b/tools/tests/run_all.py @@ -0,0 +1,22 @@ +#!/usr/bin/python + +""" +Copyright 2013 Google Inc. + +Use of this source code is governed by a BSD-style license that can be +found in the LICENSE file. + +Run all self-tests that were written in Python, raising an exception if any +of them fail. +""" + +import render_pictures_test +import skimage_self_test + +def main(): + """Run all self-tests, raising an exception if any of them fail.""" + render_pictures_test.main() + skimage_self_test.main() + +if __name__ == '__main__': + main() |