diff options
author | epoger <epoger@google.com> | 2014-06-04 15:09:18 -0700 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2014-06-04 15:09:18 -0700 |
commit | 77f68b623eeb9a50de3603388f0773a45a2604cf (patch) | |
tree | 89af3525c46b4ff02c3aea3ab5e390e82d497921 /tools/pyutils | |
parent | 30bcaa0c573032bd1f5a05f1ff270af2c56e75d1 (diff) |
Revert "rebaseline_server: download actual-results.json files from GCS instead of SVN"
This reverts commit f4666045f378dee22ce40108e90bf5195634ebfc.
BUG=skia:2641
NOTREECHECKS=True
NOTRY=True
R=jcgregorio@google.com
TBR=jcgregorio
Author: epoger@google.com
Review URL: https://codereview.chromium.org/317783004
Diffstat (limited to 'tools/pyutils')
-rw-r--r-- | tools/pyutils/__init__.py | 0 | ||||
-rwxr-xr-x | tools/pyutils/gs_utils.py | 82 | ||||
-rwxr-xr-x | tools/pyutils/url_utils.py | 63 | ||||
-rwxr-xr-x | tools/pyutils/url_utils_test.py | 61 |
4 files changed, 0 insertions, 206 deletions
diff --git a/tools/pyutils/__init__.py b/tools/pyutils/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 --- a/tools/pyutils/__init__.py +++ /dev/null diff --git a/tools/pyutils/gs_utils.py b/tools/pyutils/gs_utils.py deleted file mode 100755 index 745276e693..0000000000 --- a/tools/pyutils/gs_utils.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/python - -""" -Copyright 2014 Google Inc. - -Use of this source code is governed by a BSD-style license that can be -found in the LICENSE file. - -Utilities for accessing Google Cloud Storage. - -TODO(epoger): move this into tools/utils for broader use? -""" - -# System-level imports -import os -import posixpath -import sys - -# Imports from third-party code -TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) -APICLIENT_DIRECTORY = os.path.join( - TRUNK_DIRECTORY, 'third_party', 'externals', 'google-api-python-client') -if APICLIENT_DIRECTORY not in sys.path: - sys.path.append(APICLIENT_DIRECTORY) -from googleapiclient.discovery import build as build_service - -# Local imports -import url_utils - - -def download_file(source_bucket, source_path, dest_path, - create_subdirs_if_needed=False): - """ Downloads a single file from Google Cloud Storage to local disk. - - Args: - source_bucket: GCS bucket to download the file from - source_path: full path (Posix-style) within that bucket - dest_path: full path (local-OS-style) on local disk to copy the file to - create_subdirs_if_needed: boolean; whether to create subdirectories as - needed to create dest_path - """ - source_http_url = posixpath.join( - 'http://storage.googleapis.com', source_bucket, source_path) - url_utils.copy_contents(source_url=source_http_url, dest_path=dest_path, - create_subdirs_if_needed=create_subdirs_if_needed) - - -def list_bucket_contents(bucket, subdir=None): - """ Returns files in the Google Cloud Storage bucket as a (dirs, files) tuple. - - Uses the API documented at - https://developers.google.com/storage/docs/json_api/v1/objects/list - - Args: - bucket: name of the Google Storage bucket - subdir: directory within the bucket to list, or None for root directory - """ - # The GCS command relies on the subdir name (if any) ending with a slash. - if subdir and not subdir.endswith('/'): - subdir += '/' - subdir_length = len(subdir) if subdir else 0 - - storage = build_service('storage', 'v1') - command = storage.objects().list( - bucket=bucket, delimiter='/', fields='items(name),prefixes', - prefix=subdir) - results = command.execute() - - # The GCS command returned two subdicts: - # prefixes: the full path of every directory within subdir, with trailing '/' - # items: property dict for each file object within subdir - # (including 'name', which is full path of the object) - dirs = [] - for dir_fullpath in results.get('prefixes', []): - dir_basename = dir_fullpath[subdir_length:] - dirs.append(dir_basename[:-1]) # strip trailing slash - files = [] - for file_properties in results.get('items', []): - file_fullpath = file_properties['name'] - file_basename = file_fullpath[subdir_length:] - files.append(file_basename) - return (dirs, files) diff --git a/tools/pyutils/url_utils.py b/tools/pyutils/url_utils.py deleted file mode 100755 index b107f560db..0000000000 --- a/tools/pyutils/url_utils.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/python - -""" -Copyright 2014 Google Inc. - -Use of this source code is governed by a BSD-style license that can be -found in the LICENSE file. - -Utilities for working with URLs. - -TODO(epoger): move this into tools/utils for broader use? -""" - -# System-level imports -import contextlib -import os -import shutil -import urllib -import urlparse - - -def create_filepath_url(filepath): - """ Returns a file:/// URL pointing at the given filepath on local disk. - - Args: - filepath: string; path to a file on local disk (may be absolute or relative, - and the file does not need to exist) - - Returns: - A file:/// URL pointing at the file. Regardless of whether filepath was - specified as a relative or absolute path, the URL will contain an - absolute path to the file. - - Raises: - An Exception, if filepath is already a URL. - """ - if urlparse.urlparse(filepath).scheme: - raise Exception('"%s" is already a URL' % filepath) - return urlparse.urljoin( - 'file:', urllib.pathname2url(os.path.abspath(filepath))) - - -def copy_contents(source_url, dest_path, create_subdirs_if_needed=False): - """ Copies the full contents of the URL 'source_url' into - filepath 'dest_path'. - - Args: - source_url: string; complete URL to read from - dest_path: string; complete filepath to write to (may be absolute or - relative) - create_subdirs_if_needed: boolean; whether to create subdirectories as - needed to create dest_path - - Raises: - Some subclass of Exception if unable to read source_url or write dest_path. - """ - if create_subdirs_if_needed: - dest_dir = os.path.dirname(dest_path) - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) - with contextlib.closing(urllib.urlopen(source_url)) as source_handle: - with open(dest_path, 'wb') as dest_handle: - shutil.copyfileobj(fsrc=source_handle, fdst=dest_handle) diff --git a/tools/pyutils/url_utils_test.py b/tools/pyutils/url_utils_test.py deleted file mode 100755 index ef3d8c8aaa..0000000000 --- a/tools/pyutils/url_utils_test.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/python - -""" -Copyright 2014 Google Inc. - -Use of this source code is governed by a BSD-style license that can be -found in the LICENSE file. - -Test url_utils.py -""" - -# System-level imports -import os -import shutil -import tempfile -import unittest -import urllib - -# Imports from within Skia -import url_utils - - -class UrlUtilsTest(unittest.TestCase): - - def test_create_filepath_url(self): - """Tests create_filepath_url(). """ - with self.assertRaises(Exception): - url_utils.create_filepath_url('http://1.2.3.4/path') - # Pass absolute filepath. - self.assertEquals( - url_utils.create_filepath_url( - '%sdir%sfile' % (os.path.sep, os.path.sep)), - 'file:///dir/file') - # Pass relative filepath. - self.assertEquals( - url_utils.create_filepath_url(os.path.join('dir', 'file')), - 'file://%s/dir/file' % urllib.pathname2url(os.getcwd())) - - def test_copy_contents(self): - """Tests copy_contents(). """ - contents = 'these are the contents' - tempdir_path = tempfile.mkdtemp() - try: - source_path = os.path.join(tempdir_path, 'source') - source_url = url_utils.create_filepath_url(source_path) - with open(source_path, 'w') as source_handle: - source_handle.write(contents) - dest_path = os.path.join(tempdir_path, 'new_subdir', 'dest') - # Destination subdir does not exist, so copy_contents() should fail - # if create_subdirs_if_needed is False. - with self.assertRaises(Exception): - url_utils.copy_contents(source_url=source_url, - dest_path=dest_path, - create_subdirs_if_needed=False) - # If create_subdirs_if_needed is True, it should work. - url_utils.copy_contents(source_url=source_url, - dest_path=dest_path, - create_subdirs_if_needed=True) - self.assertEquals(open(dest_path).read(), contents) - finally: - shutil.rmtree(tempdir_path) |