reland "rebaseline_server: download actual-results.json files from GCS instead of SVN"

relands https://codereview.chromium.org/310093003 with modifications.

BUG=skia:2641
R=jcgregorio@google.com

Author: epoger@google.com

Review URL: https://codereview.chromium.org/313343003

8 files changed, 238 insertions, 23 deletions

diff --git a/tools/pyutils/__init__.py b/tools/pyutils/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tools/pyutils/__init__.py
diff --git a/tools/pyutils/gs_utils.py b/tools/pyutils/gs_utils.py
new file mode 100755
index 0000000000..2659c03e63
--- /dev/null
+++ b/tools/pyutils/gs_utils.py
@@ -0,0 +1,81 @@
+#!/usr/bin/python
+
+"""
+Copyright 2014 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+
+Utilities for accessing Google Cloud Storage.
+
+TODO(epoger): move this into tools/utils for broader use?
+"""
+
+# System-level imports
+import os
+import posixpath
+import sys
+try:
+  from apiclient.discovery import build as build_service
+except ImportError:
+  print ('Missing google-api-python-client.  Please install it; directions '
+         'can be found at https://developers.google.com/api-client-library/'
+         'python/start/installation')
+  raise
+
+# Local imports
+import url_utils
+
+
+def download_file(source_bucket, source_path, dest_path,
+                  create_subdirs_if_needed=False):
+  """ Downloads a single file from Google Cloud Storage to local disk.
+
+  Args:
+    source_bucket: GCS bucket to download the file from
+    source_path: full path (Posix-style) within that bucket
+    dest_path: full path (local-OS-style) on local disk to copy the file to
+    create_subdirs_if_needed: boolean; whether to create subdirectories as
+        needed to create dest_path
+  """
+  source_http_url = posixpath.join(
+      'http://storage.googleapis.com', source_bucket, source_path)
+  url_utils.copy_contents(source_url=source_http_url, dest_path=dest_path,
+                          create_subdirs_if_needed=create_subdirs_if_needed)
+
+
+def list_bucket_contents(bucket, subdir=None):
+  """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple.
+
+  Uses the API documented at
+  https://developers.google.com/storage/docs/json_api/v1/objects/list
+
+  Args:
+    bucket: name of the Google Storage bucket
+    subdir: directory within the bucket to list, or None for root directory
+  """
+  # The GCS command relies on the subdir name (if any) ending with a slash.
+  if subdir and not subdir.endswith('/'):
+    subdir += '/'
+  subdir_length = len(subdir) if subdir else 0
+
+  storage = build_service('storage', 'v1')
+  command = storage.objects().list(
+      bucket=bucket, delimiter='/', fields='items(name),prefixes',
+      prefix=subdir)
+  results = command.execute()
+
+  # The GCS command returned two subdicts:
+  # prefixes: the full path of every directory within subdir, with trailing '/'
+  # items: property dict for each file object within subdir
+  #        (including 'name', which is full path of the object)
+  dirs = []
+  for dir_fullpath in results.get('prefixes', []):
+    dir_basename = dir_fullpath[subdir_length:]
+    dirs.append(dir_basename[:-1])  # strip trailing slash
+  files = []
+  for file_properties in results.get('items', []):
+    file_fullpath = file_properties['name']
+    file_basename = file_fullpath[subdir_length:]
+    files.append(file_basename)
+  return (dirs, files)
diff --git a/tools/pyutils/url_utils.py b/tools/pyutils/url_utils.py
new file mode 100755
index 0000000000..b107f560db
--- /dev/null
+++ b/tools/pyutils/url_utils.py
@@ -0,0 +1,63 @@
+#!/usr/bin/python
+
+"""
+Copyright 2014 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+
+Utilities for working with URLs.
+
+TODO(epoger): move this into tools/utils for broader use?
+"""
+
+# System-level imports
+import contextlib
+import os
+import shutil
+import urllib
+import urlparse
+
+
+def create_filepath_url(filepath):
+  """ Returns a file:/// URL pointing at the given filepath on local disk.
+
+  Args:
+    filepath: string; path to a file on local disk (may be absolute or relative,
+        and the file does not need to exist)
+
+  Returns:
+    A file:/// URL pointing at the file.  Regardless of whether filepath was
+        specified as a relative or absolute path, the URL will contain an
+        absolute path to the file.
+
+  Raises:
+    An Exception, if filepath is already a URL.
+  """
+  if urlparse.urlparse(filepath).scheme:
+    raise Exception('"%s" is already a URL' % filepath)
+  return urlparse.urljoin(
+      'file:', urllib.pathname2url(os.path.abspath(filepath)))
+
+
+def copy_contents(source_url, dest_path, create_subdirs_if_needed=False):
+  """ Copies the full contents of the URL 'source_url' into
+  filepath 'dest_path'.
+
+  Args:
+    source_url: string; complete URL to read from
+    dest_path: string; complete filepath to write to (may be absolute or
+        relative)
+    create_subdirs_if_needed: boolean; whether to create subdirectories as
+        needed to create dest_path
+
+  Raises:
+    Some subclass of Exception if unable to read source_url or write dest_path.
+  """
+  if create_subdirs_if_needed:
+    dest_dir = os.path.dirname(dest_path)
+    if not os.path.exists(dest_dir):
+      os.makedirs(dest_dir)
+  with contextlib.closing(urllib.urlopen(source_url)) as source_handle:
+    with open(dest_path, 'wb') as dest_handle:
+      shutil.copyfileobj(fsrc=source_handle, fdst=dest_handle)
diff --git a/tools/pyutils/url_utils_test.py b/tools/pyutils/url_utils_test.py
new file mode 100755
index 0000000000..ef3d8c8aaa
--- /dev/null
+++ b/tools/pyutils/url_utils_test.py
@@ -0,0 +1,61 @@
+#!/usr/bin/python
+
+"""
+Copyright 2014 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+
+Test url_utils.py
+"""
+
+# System-level imports
+import os
+import shutil
+import tempfile
+import unittest
+import urllib
+
+# Imports from within Skia
+import url_utils
+
+
+class UrlUtilsTest(unittest.TestCase):
+
+  def test_create_filepath_url(self):
+    """Tests create_filepath_url(). """
+    with self.assertRaises(Exception):
+      url_utils.create_filepath_url('http://1.2.3.4/path')
+    # Pass absolute filepath.
+    self.assertEquals(
+        url_utils.create_filepath_url(
+            '%sdir%sfile' % (os.path.sep, os.path.sep)),
+        'file:///dir/file')
+    # Pass relative filepath.
+    self.assertEquals(
+        url_utils.create_filepath_url(os.path.join('dir', 'file')),
+        'file://%s/dir/file' % urllib.pathname2url(os.getcwd()))
+
+  def test_copy_contents(self):
+    """Tests copy_contents(). """
+    contents = 'these are the contents'
+    tempdir_path = tempfile.mkdtemp()
+    try:
+      source_path = os.path.join(tempdir_path, 'source')
+      source_url = url_utils.create_filepath_url(source_path)
+      with open(source_path, 'w') as source_handle:
+        source_handle.write(contents)
+      dest_path = os.path.join(tempdir_path, 'new_subdir', 'dest')
+      # Destination subdir does not exist, so copy_contents() should fail
+      # if create_subdirs_if_needed is False.
+      with self.assertRaises(Exception):
+        url_utils.copy_contents(source_url=source_url,
+                                dest_path=dest_path,
+                                create_subdirs_if_needed=False)
+      # If create_subdirs_if_needed is True, it should work.
+      url_utils.copy_contents(source_url=source_url,
+                              dest_path=dest_path,
+                              create_subdirs_if_needed=True)
+      self.assertEquals(open(dest_path).read(), contents)
+    finally:
+      shutil.rmtree(tempdir_path)
diff --git a/tools/test_all.py b/tools/test_all.py
new file mode 100755
index 0000000000..6467a2160b
--- /dev/null
+++ b/tools/test_all.py
@@ -0,0 +1,32 @@
+#!/usr/bin/python
+
+"""
+Copyright 2014 Google Inc.
+
+Use of this source code is governed by a BSD-style license that can be
+found in the LICENSE file.
+
+Run all unittests within this directory tree, recursing into subdirectories.
+"""
+
+import os
+import unittest
+
+from tests import skimage_self_test
+
+
+def main():
+  # First, run any tests that cannot be automatically discovered (because
+  # they don't use Python's unittest framework).
+  skimage_self_test.main()
+
+  # Now discover/run all tests that use Python's unittest framework.
+  suite = unittest.TestLoader().discover(os.path.dirname(__file__),
+                                         pattern='*_test.py')
+  results = unittest.TextTestRunner(verbosity=2).run(suite)
+  print repr(results)
+  if not results.wasSuccessful():
+    raise Exception('failed one or more unittests')
+
+if __name__ == '__main__':
+  main()
diff --git a/tools/tests/__init__.py b/tools/tests/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tools/tests/__init__.py
diff --git a/tools/tests/run.sh b/tools/tests/run.sh
index 6fca3cb6a2..fd12711f49 100755
--- a/tools/tests/run.sh
+++ b/tools/tests/run.sh
@@ -201,7 +201,7 @@ jsondiff_test "$JSONDIFF_INPUT/old.json $JSONDIFF_INPUT/new.json" "$JSONDIFF_OUT
 # ('make tools/tests/run.sh work cross-platform')
 #
 
-COMMAND="python tools/tests/run_all.py"
+COMMAND="python tools/test_all.py"
 echo "$COMMAND"
 $COMMAND
 ret=$?
diff --git a/tools/tests/run_all.py b/tools/tests/run_all.py
deleted file mode 100755
index 84886f758f..0000000000
--- a/tools/tests/run_all.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/python
-
-"""
-Copyright 2013 Google Inc.
-
-Use of this source code is governed by a BSD-style license that can be
-found in the LICENSE file.
-
-Run all self-tests that were written in Python, raising an exception if any
-of them fail.
-"""
-
-import render_pictures_test
-import skimage_self_test
-
-def main():
-  """Run all self-tests, raising an exception if any of them fail."""
-  render_pictures_test.main()
-  skimage_self_test.main()
-
-if __name__ == '__main__':
-  main()