1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
|
#!/usr/bin/python
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Download a file from a URL to a file on disk.
This module supports username and password with basic authentication.
"""
import base64
import os
import os.path
import sys
import urllib2
import download_utils
def _CreateDirectory(path):
"""Create a directory tree, ignore if it's already there."""
try:
os.makedirs(path)
return True
except os.error:
return False
def HttpDownload(url, target, username=None, password=None, verbose=True,
logger=None):
"""Download a file from a remote server.
Args:
url: A URL to download from.
target: Filename to write download to.
username: Optional username for download.
password: Optional password for download (ignored if no username).
logger: Function to log events to.
"""
# Log to stdout by default.
if logger is None:
logger = sys.stdout.write
headers = [('Accept', '*/*')]
if username:
if password:
auth_code = base64.b64encode(username + ':' + password)
else:
auth_code = base64.b64encode(username)
headers.append(('Authorization', 'Basic ' + auth_code))
if os.environ.get('http_proxy'):
proxy = os.environ.get('http_proxy')
proxy_handler = urllib2.ProxyHandler({
'http': proxy,
'https': proxy})
opener = urllib2.build_opener(proxy_handler)
else:
opener = urllib2.build_opener()
opener.addheaders = headers
urllib2.install_opener(opener)
_CreateDirectory(os.path.split(target)[0])
# Retry up to 10 times (appengine logger is flaky).
for i in xrange(10):
if i:
logger('Download failed on %s, retrying... (%d)\n' % (url, i))
try:
# 30 second timeout to ensure we fail and retry on stalled connections.
src = urllib2.urlopen(url, timeout=30)
try:
download_utils.WriteDataFromStream(target, src, chunk_size=2**20,
verbose=verbose)
content_len = src.headers.get('Content-Length')
if content_len:
content_len = int(content_len)
file_size = os.path.getsize(target)
if content_len != file_size:
logger('Filesize:%d does not match Content-Length:%d' % (
file_size, content_len))
continue
finally:
src.close()
break
except urllib2.HTTPError, e:
if e.code == 404:
logger('Resource does not exist.\n')
raise
logger('Failed to open.\n')
except urllib2.URLError:
logger('Failed mid stream.\n')
else:
logger('Download failed on %s, giving up.\n' % url)
raise
|
