diff options
Diffstat (limited to 'platform_tools/android/bin/download_utils.py')
-rwxr-xr-x | platform_tools/android/bin/download_utils.py | 323 |
1 files changed, 323 insertions, 0 deletions
diff --git a/platform_tools/android/bin/download_utils.py b/platform_tools/android/bin/download_utils.py new file mode 100755 index 0000000000..298ba9a863 --- /dev/null +++ b/platform_tools/android/bin/download_utils.py @@ -0,0 +1,323 @@ +#!/usr/bin/python +# Copyright (c) 2012 The Native Client Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""A library to assist automatically downloading files. + +This library is used by scripts that download tarballs, zipfiles, etc. as part +of the build process. +""" + +import hashlib +import http_download +import os.path +import re +import shutil +import sys +import time +import urllib2 + +SOURCE_STAMP = 'SOURCE_URL' +HASH_STAMP = 'SOURCE_SHA1' + + +# Designed to handle more general inputs than sys.platform because the platform +# name may come from the command line. +PLATFORM_COLLAPSE = { + 'windows': 'windows', + 'win32': 'windows', + 'cygwin': 'windows', + 'linux': 'linux', + 'linux2': 'linux', + 'linux3': 'linux', + 'darwin': 'mac', + 'mac': 'mac', +} + +ARCH_COLLAPSE = { + 'i386' : 'x86', + 'i686' : 'x86', + 'x86_64': 'x86', + 'armv7l': 'arm', +} + + +class HashError(Exception): + def __init__(self, download_url, expected_hash, actual_hash): + self.download_url = download_url + self.expected_hash = expected_hash + self.actual_hash = actual_hash + + def __str__(self): + return 'Got hash "%s" but expected hash "%s" for "%s"' % ( + self.actual_hash, self.expected_hash, self.download_url) + + +def PlatformName(name=None): + if name is None: + name = sys.platform + return PLATFORM_COLLAPSE[name] + +def ArchName(name=None): + if name is None: + if PlatformName() == 'windows': + # TODO(pdox): Figure out how to auto-detect 32-bit vs 64-bit Windows. + name = 'i386' + else: + import platform + name = platform.machine() + return ARCH_COLLAPSE[name] + +def EnsureFileCanBeWritten(filename): + directory = os.path.dirname(filename) + if not os.path.exists(directory): + os.makedirs(directory) + + +def WriteData(filename, data): + EnsureFileCanBeWritten(filename) + f = open(filename, 'wb') + f.write(data) + f.close() + + +def WriteDataFromStream(filename, stream, chunk_size, verbose=True): + EnsureFileCanBeWritten(filename) + dst = open(filename, 'wb') + try: + while True: + data = stream.read(chunk_size) + if len(data) == 0: + break + dst.write(data) + if verbose: + # Indicate that we're still writing. + sys.stdout.write('.') + sys.stdout.flush() + finally: + if verbose: + sys.stdout.write('\n') + dst.close() + + +def DoesStampMatch(stampfile, expected, index): + try: + f = open(stampfile, 'r') + stamp = f.read() + f.close() + if stamp.split('\n')[index] == expected: + return "already up-to-date." + elif stamp.startswith('manual'): + return "manual override." + return False + except IOError: + return False + + +def WriteStamp(stampfile, data): + EnsureFileCanBeWritten(stampfile) + f = open(stampfile, 'w') + f.write(data) + f.close() + + +def StampIsCurrent(path, stamp_name, stamp_contents, min_time=None, index=0): + stampfile = os.path.join(path, stamp_name) + + # Check if the stampfile is older than the minimum last mod time + if min_time: + try: + stamp_time = os.stat(stampfile).st_mtime + if stamp_time <= min_time: + return False + except OSError: + return False + + return DoesStampMatch(stampfile, stamp_contents, index) + + +def WriteSourceStamp(path, url): + stampfile = os.path.join(path, SOURCE_STAMP) + WriteStamp(stampfile, url) + +def WriteHashStamp(path, hash_val): + hash_stampfile = os.path.join(path, HASH_STAMP) + WriteStamp(hash_stampfile, hash_val) + + +def Retry(op, *args): + # Windows seems to be prone to having commands that delete files or + # directories fail. We currently do not have a complete understanding why, + # and as a workaround we simply retry the command a few times. + # It appears that file locks are hanging around longer than they should. This + # may be a secondary effect of processes hanging around longer than they + # should. This may be because when we kill a browser sel_ldr does not exit + # immediately, etc. + # Virus checkers can also accidently prevent files from being deleted, but + # that shouldn't be a problem on the bots. + if sys.platform in ('win32', 'cygwin'): + count = 0 + while True: + try: + op(*args) + break + except Exception: + sys.stdout.write("FAILED: %s %s\n" % (op.__name__, repr(args))) + count += 1 + if count < 5: + sys.stdout.write("RETRY: %s %s\n" % (op.__name__, repr(args))) + time.sleep(pow(2, count)) + else: + # Don't mask the exception. + raise + else: + op(*args) + + +def MoveDirCleanly(src, dst): + RemoveDir(dst) + MoveDir(src, dst) + + +def MoveDir(src, dst): + Retry(shutil.move, src, dst) + + +def RemoveDir(path): + if os.path.exists(path): + Retry(shutil.rmtree, path) + + +def RemoveFile(path): + if os.path.exists(path): + Retry(os.unlink, path) + + +def _HashFileHandle(fh): + """sha1 of a file like object. + + Arguments: + fh: file handle like object to hash. + Returns: + sha1 as a string. + """ + hasher = hashlib.sha1() + try: + while True: + data = fh.read(4096) + if not data: + break + hasher.update(data) + finally: + fh.close() + return hasher.hexdigest() + + +def HashFile(filename): + """sha1 a file on disk. + + Arguments: + filename: filename to hash. + Returns: + sha1 as a string. + """ + fh = open(filename, 'rb') + return _HashFileHandle(fh) + + +def HashUrlByDownloading(url): + """sha1 the data at an url. + + Arguments: + url: url to download from. + Returns: + sha1 of the data at the url. + """ + try: + fh = urllib2.urlopen(url) + except: + sys.stderr.write("Failed fetching URL: %s\n" % url) + raise + return _HashFileHandle(fh) + + +# Attempts to get the SHA1 hash of a file given a URL by looking for +# an adjacent file with a ".sha1hash" suffix. This saves having to +# download a large tarball just to get its hash. Otherwise, we fall +# back to downloading the main file. +def HashUrl(url): + hash_url = '%s.sha1hash' % url + try: + fh = urllib2.urlopen(hash_url) + data = fh.read(100) + fh.close() + except urllib2.HTTPError, exn: + if exn.code == 404: + return HashUrlByDownloading(url) + raise + else: + if not re.match('[0-9a-f]{40}\n?$', data): + raise AssertionError('Bad SHA1 hash file: %r' % data) + return data.strip() + + +def SyncURL(url, filename=None, stamp_dir=None, min_time=None, + hash_val=None, keep=False, verbose=False, stamp_index=0): + """Synchronize a destination file with a URL + + if the URL does not match the URL stamp, then we must re-download it. + + Arugments: + url: the url which will to compare against and download + filename: the file to create on download + path: the download path + stamp_dir: the filename containing the URL stamp to check against + hash_val: if set, the expected hash which must be matched + verbose: prints out status as it runs + stamp_index: index within the stamp file to check. + Returns: + True if the file is replaced + False if the file is not replaced + Exception: + HashError: if the hash does not match + """ + + assert url and filename + + # If we are not keeping the tarball, or we already have it, we can + # skip downloading it for this reason. If we are keeping it, + # it must exist. + if keep: + tarball_ok = os.path.isfile(filename) + else: + tarball_ok = True + + # If we don't need the tarball and the stamp_file matches the url, then + # we must be up to date. If the URL differs but the recorded hash matches + # the one we'll insist the tarball has, then that's good enough too. + # TODO(mcgrathr): Download the .sha1sum file first to compare with + # the cached hash, in case --file-hash options weren't used. + if tarball_ok and stamp_dir is not None: + if StampIsCurrent(stamp_dir, SOURCE_STAMP, url, min_time): + if verbose: + print '%s is already up to date.' % filename + return False + if (hash_val is not None and + StampIsCurrent(stamp_dir, HASH_STAMP, hash_val, min_time, stamp_index)): + if verbose: + print '%s is identical to the up to date file.' % filename + return False + + if verbose: + print 'Updating %s\n\tfrom %s.' % (filename, url) + EnsureFileCanBeWritten(filename) + http_download.HttpDownload(url, filename) + + if hash_val: + tar_hash = HashFile(filename) + if hash_val != tar_hash: + raise HashError(actual_hash=tar_hash, expected_hash=hash_val, + download_url=url) + + return True |