From b144271179aaf82cb1151e9dfd8e866747402594 Mon Sep 17 00:00:00 2001 From: epoger Date: Thu, 5 Jun 2014 10:30:37 -0700 Subject: reland "rebaseline_server: download actual-results.json files from GCS instead of SVN" relands https://codereview.chromium.org/310093003 with modifications. BUG=skia:2641 R=jcgregorio@google.com Author: epoger@google.com Review URL: https://codereview.chromium.org/313343003 --- tools/pyutils/__init__.py | 0 tools/pyutils/gs_utils.py | 81 +++++++++++++++++++++++++++++++++++++++++ tools/pyutils/url_utils.py | 63 ++++++++++++++++++++++++++++++++ tools/pyutils/url_utils_test.py | 61 +++++++++++++++++++++++++++++++ 4 files changed, 205 insertions(+) create mode 100644 tools/pyutils/__init__.py create mode 100755 tools/pyutils/gs_utils.py create mode 100755 tools/pyutils/url_utils.py create mode 100755 tools/pyutils/url_utils_test.py (limited to 'tools/pyutils') diff --git a/tools/pyutils/__init__.py b/tools/pyutils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/pyutils/gs_utils.py b/tools/pyutils/gs_utils.py new file mode 100755 index 0000000000..2659c03e63 --- /dev/null +++ b/tools/pyutils/gs_utils.py @@ -0,0 +1,81 @@ +#!/usr/bin/python + +""" +Copyright 2014 Google Inc. + +Use of this source code is governed by a BSD-style license that can be +found in the LICENSE file. + +Utilities for accessing Google Cloud Storage. + +TODO(epoger): move this into tools/utils for broader use? +""" + +# System-level imports +import os +import posixpath +import sys +try: + from apiclient.discovery import build as build_service +except ImportError: + print ('Missing google-api-python-client. Please install it; directions ' + 'can be found at https://developers.google.com/api-client-library/' + 'python/start/installation') + raise + +# Local imports +import url_utils + + +def download_file(source_bucket, source_path, dest_path, + create_subdirs_if_needed=False): + """ Downloads a single file from Google Cloud Storage to local disk. + + Args: + source_bucket: GCS bucket to download the file from + source_path: full path (Posix-style) within that bucket + dest_path: full path (local-OS-style) on local disk to copy the file to + create_subdirs_if_needed: boolean; whether to create subdirectories as + needed to create dest_path + """ + source_http_url = posixpath.join( + 'http://storage.googleapis.com', source_bucket, source_path) + url_utils.copy_contents(source_url=source_http_url, dest_path=dest_path, + create_subdirs_if_needed=create_subdirs_if_needed) + + +def list_bucket_contents(bucket, subdir=None): + """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple. + + Uses the API documented at + https://developers.google.com/storage/docs/json_api/v1/objects/list + + Args: + bucket: name of the Google Storage bucket + subdir: directory within the bucket to list, or None for root directory + """ + # The GCS command relies on the subdir name (if any) ending with a slash. + if subdir and not subdir.endswith('/'): + subdir += '/' + subdir_length = len(subdir) if subdir else 0 + + storage = build_service('storage', 'v1') + command = storage.objects().list( + bucket=bucket, delimiter='/', fields='items(name),prefixes', + prefix=subdir) + results = command.execute() + + # The GCS command returned two subdicts: + # prefixes: the full path of every directory within subdir, with trailing '/' + # items: property dict for each file object within subdir + # (including 'name', which is full path of the object) + dirs = [] + for dir_fullpath in results.get('prefixes', []): + dir_basename = dir_fullpath[subdir_length:] + dirs.append(dir_basename[:-1]) # strip trailing slash + files = [] + for file_properties in results.get('items', []): + file_fullpath = file_properties['name'] + file_basename = file_fullpath[subdir_length:] + files.append(file_basename) + return (dirs, files) diff --git a/tools/pyutils/url_utils.py b/tools/pyutils/url_utils.py new file mode 100755 index 0000000000..b107f560db --- /dev/null +++ b/tools/pyutils/url_utils.py @@ -0,0 +1,63 @@ +#!/usr/bin/python + +""" +Copyright 2014 Google Inc. + +Use of this source code is governed by a BSD-style license that can be +found in the LICENSE file. + +Utilities for working with URLs. + +TODO(epoger): move this into tools/utils for broader use? +""" + +# System-level imports +import contextlib +import os +import shutil +import urllib +import urlparse + + +def create_filepath_url(filepath): + """ Returns a file:/// URL pointing at the given filepath on local disk. + + Args: + filepath: string; path to a file on local disk (may be absolute or relative, + and the file does not need to exist) + + Returns: + A file:/// URL pointing at the file. Regardless of whether filepath was + specified as a relative or absolute path, the URL will contain an + absolute path to the file. + + Raises: + An Exception, if filepath is already a URL. + """ + if urlparse.urlparse(filepath).scheme: + raise Exception('"%s" is already a URL' % filepath) + return urlparse.urljoin( + 'file:', urllib.pathname2url(os.path.abspath(filepath))) + + +def copy_contents(source_url, dest_path, create_subdirs_if_needed=False): + """ Copies the full contents of the URL 'source_url' into + filepath 'dest_path'. + + Args: + source_url: string; complete URL to read from + dest_path: string; complete filepath to write to (may be absolute or + relative) + create_subdirs_if_needed: boolean; whether to create subdirectories as + needed to create dest_path + + Raises: + Some subclass of Exception if unable to read source_url or write dest_path. + """ + if create_subdirs_if_needed: + dest_dir = os.path.dirname(dest_path) + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + with contextlib.closing(urllib.urlopen(source_url)) as source_handle: + with open(dest_path, 'wb') as dest_handle: + shutil.copyfileobj(fsrc=source_handle, fdst=dest_handle) diff --git a/tools/pyutils/url_utils_test.py b/tools/pyutils/url_utils_test.py new file mode 100755 index 0000000000..ef3d8c8aaa --- /dev/null +++ b/tools/pyutils/url_utils_test.py @@ -0,0 +1,61 @@ +#!/usr/bin/python + +""" +Copyright 2014 Google Inc. + +Use of this source code is governed by a BSD-style license that can be +found in the LICENSE file. + +Test url_utils.py +""" + +# System-level imports +import os +import shutil +import tempfile +import unittest +import urllib + +# Imports from within Skia +import url_utils + + +class UrlUtilsTest(unittest.TestCase): + + def test_create_filepath_url(self): + """Tests create_filepath_url(). """ + with self.assertRaises(Exception): + url_utils.create_filepath_url('http://1.2.3.4/path') + # Pass absolute filepath. + self.assertEquals( + url_utils.create_filepath_url( + '%sdir%sfile' % (os.path.sep, os.path.sep)), + 'file:///dir/file') + # Pass relative filepath. + self.assertEquals( + url_utils.create_filepath_url(os.path.join('dir', 'file')), + 'file://%s/dir/file' % urllib.pathname2url(os.getcwd())) + + def test_copy_contents(self): + """Tests copy_contents(). """ + contents = 'these are the contents' + tempdir_path = tempfile.mkdtemp() + try: + source_path = os.path.join(tempdir_path, 'source') + source_url = url_utils.create_filepath_url(source_path) + with open(source_path, 'w') as source_handle: + source_handle.write(contents) + dest_path = os.path.join(tempdir_path, 'new_subdir', 'dest') + # Destination subdir does not exist, so copy_contents() should fail + # if create_subdirs_if_needed is False. + with self.assertRaises(Exception): + url_utils.copy_contents(source_url=source_url, + dest_path=dest_path, + create_subdirs_if_needed=False) + # If create_subdirs_if_needed is True, it should work. + url_utils.copy_contents(source_url=source_url, + dest_path=dest_path, + create_subdirs_if_needed=True) + self.assertEquals(open(dest_path).read(), contents) + finally: + shutil.rmtree(tempdir_path) -- cgit v1.2.3