aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorGravatar Brennan Saeta <saeta@google.com>2017-12-01 11:08:51 -0800
committerGravatar TensorFlower Gardener <gardener@tensorflow.org>2017-12-01 11:12:11 -0800
commitd8e98c8753e3d89e6d4e989e52e910495c4f64ac (patch)
treee411fa19f1512a376f92c4acd4e49aed41930d79
parent11749434e3eb04eee058a43a931a27bdee4916df (diff)
Customize request timeouts for GCS filesystem.
PiperOrigin-RevId: 177614538
-rw-r--r--tensorflow/core/platform/cloud/curl_http_request.cc29
-rw-r--r--tensorflow/core/platform/cloud/curl_http_request.h12
-rw-r--r--tensorflow/core/platform/cloud/gcs_dns_cache_test.cc5
-rw-r--r--tensorflow/core/platform/cloud/gcs_file_system.cc65
-rw-r--r--tensorflow/core/platform/cloud/gcs_file_system.h42
-rw-r--r--tensorflow/core/platform/cloud/gcs_file_system_test.cc649
-rw-r--r--tensorflow/core/platform/cloud/http_request.h10
-rw-r--r--tensorflow/core/platform/cloud/http_request_fake.h23
8 files changed, 572 insertions, 263 deletions
diff --git a/tensorflow/core/platform/cloud/curl_http_request.cc b/tensorflow/core/platform/cloud/curl_http_request.cc
index 4581a0870a..f7fbfe971e 100644
--- a/tensorflow/core/platform/cloud/curl_http_request.cc
+++ b/tensorflow/core/platform/cloud/curl_http_request.cc
@@ -29,16 +29,6 @@ namespace {
// Set to 1 to enable verbose debug output from curl.
constexpr uint64 kVerboseOutput = 0;
-// Timeout for the whole request. Set only to prevent hanging indefinitely.
-constexpr uint32 kRequestTimeoutSeconds = 3600; // 1 hour
-
-// Timeout for the connection phase.
-constexpr uint32 kConnectTimeoutSeconds = 120; // 2 minutes
-
-// The maximum period of request inactivity, after which the request
-// is terminated.
-constexpr uint64 kInactivityTimeoutSeconds = 60; // 1 minute
-
// Proxy to the real libcurl implementation.
class LibCurlProxy : public LibCurl {
public:
@@ -161,9 +151,6 @@ Status CurlHttpRequest::Init() {
strings::StrCat("TensorFlow/", TF_VERSION_STRING).c_str());
// Do not use signals for timeouts - does not work in multi-threaded programs.
libcurl_->curl_easy_setopt(curl_, CURLOPT_NOSIGNAL, 1L);
- libcurl_->curl_easy_setopt(curl_, CURLOPT_TIMEOUT, kRequestTimeoutSeconds);
- libcurl_->curl_easy_setopt(curl_, CURLOPT_CONNECTTIMEOUT,
- kConnectTimeoutSeconds);
libcurl_->curl_easy_setopt(curl_, CURLOPT_HTTP_VERSION,
CURL_HTTP_VERSION_2_0);
@@ -336,6 +323,16 @@ Status CurlHttpRequest::SetResultBuffer(std::vector<char>* out_buffer) {
return Status::OK();
}
+Status CurlHttpRequest::SetTimeouts(uint32 connection, uint32 inactivity,
+ uint32 total) {
+ TF_RETURN_IF_ERROR(CheckInitialized());
+ TF_RETURN_IF_ERROR(CheckNotSent());
+ connect_timeout_secs_ = connection;
+ inactivity_timeout_secs_ = inactivity;
+ request_timeout_secs_ = total;
+ return Status::OK();
+}
+
size_t CurlHttpRequest::WriteCallback(const void* ptr, size_t size,
size_t nmemb, void* this_object) {
CHECK(ptr);
@@ -399,6 +396,10 @@ Status CurlHttpRequest::Send() {
libcurl_->curl_easy_setopt(curl_, CURLOPT_HEADERFUNCTION,
&CurlHttpRequest::HeaderCallback);
+ libcurl_->curl_easy_setopt(curl_, CURLOPT_TIMEOUT, request_timeout_secs_);
+ libcurl_->curl_easy_setopt(curl_, CURLOPT_CONNECTTIMEOUT,
+ connect_timeout_secs_);
+
char error_buffer[CURL_ERROR_SIZE] = {0};
libcurl_->curl_easy_setopt(curl_, CURLOPT_ERRORBUFFER, error_buffer);
@@ -529,7 +530,7 @@ int CurlHttpRequest::ProgressCallback(void* this_object, curl_off_t dltotal,
return 0;
}
- if (now - that->last_progress_timestamp_ > kInactivityTimeoutSeconds) {
+ if (now - that->last_progress_timestamp_ > that->inactivity_timeout_secs_) {
LOG(ERROR) << "The transmission of request " << this_object
<< " (URI: " << that->uri_ << ") has been stuck at "
<< current_progress << " of " << dltotal + ultotal
diff --git a/tensorflow/core/platform/cloud/curl_http_request.h b/tensorflow/core/platform/cloud/curl_http_request.h
index 9e5ae61016..cf249450df 100644
--- a/tensorflow/core/platform/cloud/curl_http_request.h
+++ b/tensorflow/core/platform/cloud/curl_http_request.h
@@ -120,6 +120,9 @@ class CurlHttpRequest : public HttpRequest {
// Url encodes str and returns a new string.
string EscapeString(const string& str) override;
+ Status SetTimeouts(uint32 connection, uint32 inactivity,
+ uint32 total) override;
+
private:
/// A write callback in the form which can be accepted by libcurl.
static size_t WriteCallback(const void* ptr, size_t size, size_t nmemb,
@@ -162,6 +165,15 @@ class CurlHttpRequest : public HttpRequest {
// The last progress in terms of bytes transmitted.
curl_off_t last_progress_bytes_ = 0;
+ // The maximum period of request inactivity.
+ uint32 inactivity_timeout_secs_ = 60; // 1 minute
+
+ // Timeout for the connection phase.
+ uint32 connect_timeout_secs_ = 120; // 2 minutes
+
+ // Tiemout for the whole request. Set only to prevent hanging indefinitely.
+ uint32 request_timeout_secs_ = 3600; // 1 hour
+
// Members to enforce the usage flow.
bool is_initialized_ = false;
bool is_uri_set_ = false;
diff --git a/tensorflow/core/platform/cloud/gcs_dns_cache_test.cc b/tensorflow/core/platform/cloud/gcs_dns_cache_test.cc
index 8d1a108f30..266879ddf5 100644
--- a/tensorflow/core/platform/cloud/gcs_dns_cache_test.cc
+++ b/tensorflow/core/platform/cloud/gcs_dns_cache_test.cc
@@ -64,6 +64,11 @@ class TestHttpRequest : public HttpRequest {
Status Send() override { return Status::OK(); }
string EscapeString(const string& str) override { return ""; }
+ Status SetTimeouts(uint32 connection, uint32 inactivity,
+ uint32 total) override {
+ return Status::OK();
+ }
+
std::map<string, string> resolve_overrides_;
};
diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc
index 45e9b05092..ab82643ad5 100644
--- a/tensorflow/core/platform/cloud/gcs_file_system.cc
+++ b/tensorflow/core/platform/cloud/gcs_file_system.cc
@@ -94,6 +94,20 @@ const FileStatistics DIRECTORY_STAT(0, 0, true);
// variable to a positive integer describing the frequency used to refresh the
// userspace DNS cache.
constexpr char kResolveCacheSecs[] = "GCS_RESOLVE_REFRESH_SECS";
+// The environment variable to configure the http request's connection timeout.
+constexpr char kRequestConnectionTimeout[] =
+ "GCS_REQUEST_CONNECTION_TIMEOUT_SECS";
+// The environment varaible to configure the http request's idle timeout.
+constexpr char kRequestIdleTimeout[] = "GCS_REQUEST_IDLE_TIMEOUT_SECS";
+// The environment variable to configure the overall request timeout for
+// metadata requests.
+constexpr char kMetadataRequestTimeout[] = "GCS_METADATA_REQUEST_TIMEOUT_SECS";
+// The environment variable to configure the overall request timeout for
+// block reads requests.
+constexpr char kReadRequestTimeout[] = "GCS_READ_REQUEST_TIMEOUT_SECS";
+// The environment variable to configure the overall request timeout for
+// upload requests.
+constexpr char kWriteRequestTimeout[] = "GCS_WRITE_REQUEST_TIMEOUT_SECS";
Status GetTmpFilename(string* filename) {
if (!filename) {
@@ -283,12 +297,14 @@ class GcsWritableFile : public WritableFile {
GcsWritableFile(const string& bucket, const string& object,
AuthProvider* auth_provider,
HttpRequest::Factory* http_request_factory,
+ GcsFileSystem::TimeoutConfig* timeouts,
std::function<void()> file_cache_erase,
int64 initial_retry_delay_usec)
: bucket_(bucket),
object_(object),
auth_provider_(auth_provider),
http_request_factory_(http_request_factory),
+ timeouts_(timeouts),
file_cache_erase_(std::move(file_cache_erase)),
sync_needed_(true),
initial_retry_delay_usec_(initial_retry_delay_usec) {
@@ -307,12 +323,14 @@ class GcsWritableFile : public WritableFile {
AuthProvider* auth_provider,
const string& tmp_content_filename,
HttpRequest::Factory* http_request_factory,
+ GcsFileSystem::TimeoutConfig* timeouts,
std::function<void()> file_cache_erase,
int64 initial_retry_delay_usec)
: bucket_(bucket),
object_(object),
auth_provider_(auth_provider),
http_request_factory_(http_request_factory),
+ timeouts_(timeouts),
file_cache_erase_(std::move(file_cache_erase)),
sync_needed_(true),
initial_retry_delay_usec_(initial_retry_delay_usec) {
@@ -446,6 +464,8 @@ class GcsWritableFile : public WritableFile {
std::to_string(file_size)));
TF_RETURN_IF_ERROR(request->SetPostEmptyBody());
TF_RETURN_IF_ERROR(request->SetResultBuffer(&output_buffer));
+ TF_RETURN_IF_ERROR(request->SetTimeouts(timeouts_->connect, timeouts_->idle,
+ timeouts_->metadata));
TF_RETURN_WITH_CONTEXT_IF_ERROR(
request->Send(), " when initiating an upload to ", GetGcsPath());
*session_uri = request->GetResponseHeader("Location");
@@ -477,6 +497,8 @@ class GcsWritableFile : public WritableFile {
TF_RETURN_IF_ERROR(request->Init());
TF_RETURN_IF_ERROR(request->SetUri(session_uri));
TF_RETURN_IF_ERROR(request->AddAuthBearerHeader(auth_token));
+ TF_RETURN_IF_ERROR(request->SetTimeouts(timeouts_->connect, timeouts_->idle,
+ timeouts_->metadata));
TF_RETURN_IF_ERROR(request->AddHeader(
"Content-Range", strings::StrCat("bytes */", file_size)));
TF_RETURN_IF_ERROR(request->SetPutEmptyBody());
@@ -531,6 +553,9 @@ class GcsWritableFile : public WritableFile {
"Content-Range", strings::StrCat("bytes ", start_offset, "-",
file_size - 1, "/", file_size)));
}
+ TF_RETURN_IF_ERROR(request->SetTimeouts(timeouts_->connect, timeouts_->idle,
+ timeouts_->write));
+
TF_RETURN_IF_ERROR(
request->SetPutFromFile(tmp_content_filename_, start_offset));
TF_RETURN_WITH_CONTEXT_IF_ERROR(request->Send(), " when uploading ",
@@ -550,6 +575,7 @@ class GcsWritableFile : public WritableFile {
string tmp_content_filename_;
std::ofstream outfile_;
HttpRequest::Factory* http_request_factory_;
+ GcsFileSystem::TimeoutConfig* timeouts_;
std::function<void()> file_cache_erase_;
bool sync_needed_; // whether there is buffered data that needs to be synced
int64 initial_retry_delay_usec_;
@@ -635,6 +661,25 @@ GcsFileSystem::GcsFileSystem()
&resolve_frequency_secs)) {
dns_cache_.reset(new GcsDnsCache(resolve_frequency_secs));
}
+ // Apply the overrides for request timeouts
+ uint32 timeout_value;
+ if (GetEnvVar(kRequestConnectionTimeout, strings::safe_strtou32,
+ &timeout_value)) {
+ timeouts_.connect = timeout_value;
+ }
+ if (GetEnvVar(kRequestIdleTimeout, strings::safe_strtou32, &timeout_value)) {
+ timeouts_.idle = timeout_value;
+ }
+ if (GetEnvVar(kMetadataRequestTimeout, strings::safe_strtou32,
+ &timeout_value)) {
+ timeouts_.metadata = timeout_value;
+ }
+ if (GetEnvVar(kReadRequestTimeout, strings::safe_strtou32, &timeout_value)) {
+ timeouts_.read = timeout_value;
+ }
+ if (GetEnvVar(kWriteRequestTimeout, strings::safe_strtou32, &timeout_value)) {
+ timeouts_.write = timeout_value;
+ }
}
GcsFileSystem::GcsFileSystem(
@@ -643,7 +688,8 @@ GcsFileSystem::GcsFileSystem(
size_t block_size, size_t max_bytes, uint64 max_staleness,
uint64 stat_cache_max_age, size_t stat_cache_max_entries,
uint64 matching_paths_cache_max_age,
- size_t matching_paths_cache_max_entries, int64 initial_retry_delay_usec)
+ size_t matching_paths_cache_max_entries, int64 initial_retry_delay_usec,
+ TimeoutConfig timeouts)
: auth_provider_(std::move(auth_provider)),
http_request_factory_(std::move(http_request_factory)),
file_block_cache_(
@@ -651,6 +697,7 @@ GcsFileSystem::GcsFileSystem(
stat_cache_(new StatCache(stat_cache_max_age, stat_cache_max_entries)),
matching_paths_cache_(new MatchingPathsCache(
matching_paths_cache_max_age, matching_paths_cache_max_entries)),
+ timeouts_(timeouts),
initial_retry_delay_usec_(initial_retry_delay_usec) {}
Status GcsFileSystem::NewRandomAccessFile(
@@ -689,6 +736,8 @@ Status GcsFileSystem::LoadBufferFromGCS(const string& filename, size_t offset,
TF_RETURN_IF_ERROR(request->AddAuthBearerHeader(auth_token));
TF_RETURN_IF_ERROR(request->SetRange(offset, offset + n - 1));
TF_RETURN_IF_ERROR(request->SetResultBuffer(out));
+ TF_RETURN_IF_ERROR(
+ request->SetTimeouts(timeouts_.connect, timeouts_.idle, timeouts_.read));
if (dns_cache_) {
TF_RETURN_IF_ERROR(dns_cache_->AnnotateRequest(request.get()));
@@ -723,7 +772,7 @@ Status GcsFileSystem::NewWritableFile(const string& fname,
TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object));
result->reset(new GcsWritableFile(
bucket, object, auth_provider_.get(), http_request_factory_.get(),
- [this, fname]() { file_block_cache_->RemoveFile(fname); },
+ &timeouts_, [this, fname]() { file_block_cache_->RemoveFile(fname); },
initial_retry_delay_usec_));
return Status::OK();
}
@@ -764,7 +813,7 @@ Status GcsFileSystem::NewAppendableFile(const string& fname,
TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object));
result->reset(new GcsWritableFile(
bucket, object, auth_provider_.get(), old_content_filename,
- http_request_factory_.get(),
+ http_request_factory_.get(), &timeouts_,
[this, fname]() { file_block_cache_->RemoveFile(fname); },
initial_retry_delay_usec_));
return Status::OK();
@@ -852,6 +901,8 @@ Status GcsFileSystem::StatForObject(const string& fname, const string& bucket,
"?fields=size%2Cupdated")));
TF_RETURN_IF_ERROR(request->AddAuthBearerHeader(auth_token));
TF_RETURN_IF_ERROR(request->SetResultBuffer(&output_buffer));
+ TF_RETURN_IF_ERROR(request->SetTimeouts(
+ timeouts_.connect, timeouts_.idle, timeouts_.metadata));
if (dns_cache_) {
TF_RETURN_IF_ERROR(dns_cache_->AnnotateRequest(request.get()));
@@ -902,6 +953,8 @@ Status GcsFileSystem::BucketExists(const string& bucket, bool* result) {
TF_RETURN_IF_ERROR(
request->SetUri(strings::StrCat(kGcsUriBase, "b/", bucket)));
TF_RETURN_IF_ERROR(request->AddAuthBearerHeader(auth_token));
+ TF_RETURN_IF_ERROR(request->SetTimeouts(timeouts_.connect, timeouts_.idle,
+ timeouts_.metadata));
const Status status = request->Send();
switch (status.code()) {
case errors::Code::OK:
@@ -1033,6 +1086,8 @@ Status GcsFileSystem::GetChildrenBounded(const string& dirname,
TF_RETURN_IF_ERROR(request->SetUri(uri));
TF_RETURN_IF_ERROR(request->AddAuthBearerHeader(auth_token));
TF_RETURN_IF_ERROR(request->SetResultBuffer(&output_buffer));
+ TF_RETURN_IF_ERROR(request->SetTimeouts(timeouts_.connect, timeouts_.idle,
+ timeouts_.metadata));
if (dns_cache_) {
TF_RETURN_IF_ERROR(dns_cache_->AnnotateRequest(request.get()));
@@ -1157,6 +1212,8 @@ Status GcsFileSystem::DeleteFile(const string& fname) {
TF_RETURN_IF_ERROR(request->SetUri(strings::StrCat(
kGcsUriBase, "b/", bucket, "/o/", request->EscapeString(object))));
TF_RETURN_IF_ERROR(request->AddAuthBearerHeader(auth_token));
+ TF_RETURN_IF_ERROR(request->SetTimeouts(timeouts_.connect, timeouts_.idle,
+ timeouts_.metadata));
TF_RETURN_IF_ERROR(request->SetDeleteRequest());
TF_RETURN_WITH_CONTEXT_IF_ERROR(request->Send(), " when deleting ", fname);
file_block_cache_->RemoveFile(fname);
@@ -1251,6 +1308,8 @@ Status GcsFileSystem::RenameObject(const string& src, const string& target) {
request->EscapeString(target_object))));
TF_RETURN_IF_ERROR(request->AddAuthBearerHeader(auth_token));
TF_RETURN_IF_ERROR(request->SetPostEmptyBody());
+ TF_RETURN_IF_ERROR(request->SetTimeouts(timeouts_.connect, timeouts_.idle,
+ timeouts_.metadata));
std::vector<char> output_buffer;
TF_RETURN_IF_ERROR(request->SetResultBuffer(&output_buffer));
TF_RETURN_WITH_CONTEXT_IF_ERROR(request->Send(), " when renaming ", src,
diff --git a/tensorflow/core/platform/cloud/gcs_file_system.h b/tensorflow/core/platform/cloud/gcs_file_system.h
index 4b4853c838..7cfcebd5c9 100644
--- a/tensorflow/core/platform/cloud/gcs_file_system.h
+++ b/tensorflow/core/platform/cloud/gcs_file_system.h
@@ -35,6 +35,8 @@ namespace tensorflow {
/// which adds retry logic to GCS operations.
class GcsFileSystem : public FileSystem {
public:
+ struct TimeoutConfig;
+
GcsFileSystem();
GcsFileSystem(std::unique_ptr<AuthProvider> auth_provider,
std::unique_ptr<HttpRequest::Factory> http_request_factory,
@@ -42,7 +44,7 @@ class GcsFileSystem : public FileSystem {
uint64 stat_cache_max_age, size_t stat_cache_max_entries,
uint64 matching_paths_cache_max_age,
size_t matching_paths_cache_max_entries,
- int64 initial_retry_delay_usec);
+ int64 initial_retry_delay_usec, TimeoutConfig timeouts);
Status NewRandomAccessFile(
const string& filename,
@@ -87,6 +89,7 @@ class GcsFileSystem : public FileSystem {
size_t block_size() const { return file_block_cache_->block_size(); }
size_t max_bytes() const { return file_block_cache_->max_bytes(); }
uint64 max_staleness() const { return file_block_cache_->max_staleness(); }
+ TimeoutConfig timeouts() const { return timeouts_; }
uint64 stat_cache_max_age() const { return stat_cache_->max_age(); }
size_t stat_cache_max_entries() const { return stat_cache_->max_entries(); }
@@ -98,6 +101,41 @@ class GcsFileSystem : public FileSystem {
return matching_paths_cache_->max_entries();
}
+ /// Structure containing the information for timeouts related to accessing the
+ /// GCS APIs.
+ ///
+ /// All values are in seconds.
+ struct TimeoutConfig {
+ // The request connection timeout. If a connection cannot be established
+ // within `connect` seconds, abort the request.
+ uint32 connect = 120; // 2 minutes
+
+ // The request idle timeout. If a request has seen no activity in `idle`
+ // seconds, abort the request.
+ uint32 idle = 60; // 1 minute
+
+ // The maximum total time a metadata request can take. If a request has not
+ // completed within `metadata` seconds, the request is aborted.
+ uint32 metadata = 3600; // 1 hour
+
+ // The maximum total time a block read request can take. If a request has
+ // not completed within `read` seconds, the request is aborted.
+ uint32 read = 3600; // 1 hour
+
+ // The maximum total time an upload request can take. If a request has not
+ // completed within `write` seconds, the request is aborted.
+ uint32 write = 3600; // 1 hour
+
+ TimeoutConfig() {}
+ TimeoutConfig(uint32 connect, uint32 idle, uint32 metadata, uint32 read,
+ uint32 write)
+ : connect(connect),
+ idle(idle),
+ metadata(metadata),
+ read(read),
+ write(write) {}
+ };
+
private:
/// \brief Checks if the bucket exists. Returns OK if the check succeeded.
///
@@ -150,6 +188,8 @@ class GcsFileSystem : public FileSystem {
using MatchingPathsCache = ExpiringLRUCache<std::vector<string>>;
std::unique_ptr<MatchingPathsCache> matching_paths_cache_;
+ TimeoutConfig timeouts_;
+
/// The initial delay for exponential backoffs when retrying failed calls.
const int64 initial_retry_delay_usec_ = 1000000L;
diff --git a/tensorflow/core/platform/cloud/gcs_file_system_test.cc b/tensorflow/core/platform/cloud/gcs_file_system_test.cc
index 7614ec4d7f..01f4fd8688 100644
--- a/tensorflow/core/platform/cloud/gcs_file_system_test.cc
+++ b/tensorflow/core/platform/cloud/gcs_file_system_test.cc
@@ -22,6 +22,8 @@ limitations under the License.
namespace tensorflow {
namespace {
+static GcsFileSystem::TimeoutConfig kTestTimeoutConfig(5, 1, 10, 20, 30);
+
class FakeAuthProvider : public AuthProvider {
public:
Status GetToken(string* token) override {
@@ -35,12 +37,14 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) {
{new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
"Auth Token: fake_token\n"
- "Range: 0-5\n",
+ "Range: 0-5\n"
+ "Timeouts: 5 1 20\n",
"012345"),
new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
"Auth Token: fake_token\n"
- "Range: 6-11\n",
+ "Range: 6-11\n"
+ "Timeouts: 5 1 20\n",
"6789")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -49,7 +53,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
std::unique_ptr<RandomAccessFile> file;
TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
@@ -73,12 +77,14 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_differentN) {
{new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
"Auth Token: fake_token\n"
- "Range: 0-2\n",
+ "Range: 0-2\n"
+ "Timeouts: 5 1 20\n",
"012"),
new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
"Auth Token: fake_token\n"
- "Range: 3-12\n",
+ "Range: 3-12\n"
+ "Timeouts: 5 1 20\n",
"3456789")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -87,7 +93,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_differentN) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
std::unique_ptr<RandomAccessFile> file;
TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
@@ -116,26 +122,30 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache) {
{new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
"Auth Token: fake_token\n"
- "Range: 0-8\n",
+ "Range: 0-8\n"
+ "Timeouts: 5 1 20\n",
"012345678"),
new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
"Auth Token: fake_token\n"
- "Range: 9-17\n",
+ "Range: 9-17\n"
+ "Timeouts: 5 1 20\n",
"9abcde"),
new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/random_access.txt\n"
"Auth Token: fake_token\n"
- "Range: 18-26\n",
+ "Range: 18-26\n"
+ "Timeouts: 5 1 20\n",
"")});
- GcsFileSystem fs(
- std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */, 0 /* initial retry delay */);
+ GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ 9 /* block size */, 18 /* max bytes */,
+ 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */,
+ 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */,
+ 0 /* initial retry delay */, kTestTimeoutConfig);
char scratch[100];
StringPiece result;
@@ -191,20 +201,23 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_MaxStaleness) {
std::vector<HttpRequest*> requests(
{new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n"
"Auth Token: fake_token\n"
- "Range: 0-7\n",
+ "Range: 0-7\n"
+ "Timeouts: 5 1 20\n",
"01234567"),
new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n"
"Auth Token: fake_token\n"
- "Range: 8-15\n",
+ "Range: 8-15\n"
+ "Timeouts: 5 1 20\n",
"89abcdef")});
- GcsFileSystem fs(
- std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 8 /* block size */, 16 /* max bytes */, 3600 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */, 0 /* initial retry delay */);
+ GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ 8 /* block size */, 16 /* max bytes */,
+ 3600 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */,
+ 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */,
+ 0 /* initial retry delay */, kTestTimeoutConfig);
char scratch[100];
StringPiece result;
// There should only be two HTTP requests issued to GCS even though we iterate
@@ -238,14 +251,15 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_MaxStaleness) {
TEST(GcsFileSystemTest, NewRandomAccessFile_NoObjectName) {
std::vector<HttpRequest*> requests;
- GcsFileSystem fs(
- std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* read ahead bytes */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */, 0 /* initial retry delay */);
+ GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ 0 /* read ahead bytes */, 0 /* max bytes */,
+ 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */,
+ 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */,
+ 0 /* initial retry delay */, kTestTimeoutConfig);
std::unique_ptr<RandomAccessFile> file;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -257,24 +271,28 @@ TEST(GcsFileSystemTest, NewWritableFile) {
{new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
"Auth Token: fake_token\n"
- "Range: 0-7\n",
+ "Range: 0-7\n"
+ "Timeouts: 5 1 20\n",
"01234567"),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
"uploadType=resumable&name=path%2Fwriteable\n"
"Auth Token: fake_token\n"
"Header X-Upload-Content-Length: 17\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"", {{"Location", "https://custom/upload/location"}}),
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
"Header Content-Range: bytes 0-16/17\n"
+ "Timeouts: 5 1 30\n"
"Put body: content1,content2\n",
""),
new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
"Auth Token: fake_token\n"
- "Range: 0-7\n",
+ "Range: 0-7\n"
+ "Timeouts: 5 1 20\n",
"01234567")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -283,7 +301,7 @@ TEST(GcsFileSystemTest, NewWritableFile) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
// Read from the file first, to fill the block cache.
std::unique_ptr<RandomAccessFile> rfile;
@@ -315,33 +333,39 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) {
"uploadType=resumable&name=path%2Fwriteable.txt\n"
"Auth Token: fake_token\n"
"Header X-Upload-Content-Length: 17\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"", {{"Location", "https://custom/upload/location"}}),
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
"Header Content-Range: bytes 0-16/17\n"
+ "Timeouts: 5 1 30\n"
"Put body: content1,content2\n",
"", errors::Unavailable("503"), 503),
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Header Content-Range: bytes */17\n"
"Put: yes\n",
"", errors::FailedPrecondition("308"), nullptr,
- {{"Range", "0-10"}}, 308),
+ {{"Range", "0-10"}}, 308, {}),
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
"Header Content-Range: bytes 11-16/17\n"
+ "Timeouts: 5 1 30\n"
"Put body: ntent2\n",
"", errors::Unavailable("503"), 503),
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Header Content-Range: bytes */17\n"
"Put: yes\n",
"", errors::FailedPrecondition("308"), nullptr,
- {{"Range", "bytes=0-12"}}, 308),
+ {{"Range", "bytes=0-12"}}, 308, {}),
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
"Header Content-Range: bytes 13-16/17\n"
+ "Timeouts: 5 1 30\n"
"Put body: ent2\n",
"")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -351,7 +375,7 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
std::unique_ptr<WritableFile> file;
TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file));
@@ -369,38 +393,44 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceedsOnGetStatus) {
{new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
"Auth Token: fake_token\n"
- "Range: 0-7\n",
+ "Range: 0-7\n"
+ "Timeouts: 5 1 20\n",
"01234567"),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
"uploadType=resumable&name=path%2Fwriteable\n"
"Auth Token: fake_token\n"
"Header X-Upload-Content-Length: 17\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"", {{"Location", "https://custom/upload/location"}}),
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
"Header Content-Range: bytes 0-16/17\n"
+ "Timeouts: 5 1 30\n"
"Put body: content1,content2\n",
"", errors::Unavailable("503"), 503),
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Header Content-Range: bytes */17\n"
"Put: yes\n",
- "", Status::OK(), nullptr, {}, 201),
+ "", Status::OK(), nullptr, {}, 201, {}),
new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
"Auth Token: fake_token\n"
- "Range: 0-7\n",
+ "Range: 0-7\n"
+ "Timeouts: 5 1 20\n",
"01234567")});
- GcsFileSystem fs(
- std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 8 /* block size */, 8 /* max bytes */, 3600 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */, 0 /* initial retry delay */);
+ GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ 8 /* block size */, 8 /* max bytes */,
+ 3600 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */,
+ 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */,
+ 0 /* initial retry delay */, kTestTimeoutConfig);
// Pull the file's first block into the cache. This will trigger the first
// HTTP request to GCS.
std::unique_ptr<RandomAccessFile> rfile;
@@ -434,25 +464,29 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) {
"uploadType=resumable&name=path%2Fwriteable.txt\n"
"Auth Token: fake_token\n"
"Header X-Upload-Content-Length: 17\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"", {{"Location", "https://custom/upload/location"}}),
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
"Header Content-Range: bytes 0-16/17\n"
+ "Timeouts: 5 1 30\n"
"Put body: content1,content2\n",
"", errors::Unavailable("503"), 503)});
for (int i = 0; i < 10; i++) {
requests.emplace_back(new FakeHttpRequest(
"Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Header Content-Range: bytes */17\n"
"Put: yes\n",
"", errors::FailedPrecondition("important HTTP error 308"), nullptr,
- {{"Range", "0-10"}}, 308));
+ {{"Range", "0-10"}}, 308, {}));
requests.emplace_back(new FakeHttpRequest(
"Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
"Header Content-Range: bytes 11-16/17\n"
+ "Timeouts: 5 1 30\n"
"Put body: ntent2\n",
"", errors::Unavailable("important HTTP error 503"), 503));
}
@@ -463,12 +497,14 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) {
"uploadType=resumable&name=path%2Fwriteable.txt\n"
"Auth Token: fake_token\n"
"Header X-Upload-Content-Length: 17\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"", {{"Location", "https://custom/upload/location"}}));
requests.emplace_back(
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
"Header Content-Range: bytes 0-16/17\n"
+ "Timeouts: 5 1 30\n"
"Put body: content1,content2\n",
""));
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -478,7 +514,7 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 2 /* initial retry delay */);
+ 2 /* initial retry delay */, kTestTimeoutConfig);
std::unique_ptr<WritableFile> file;
TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file));
@@ -500,11 +536,13 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) {
"uploadType=resumable&name=path%2Fwriteable.txt\n"
"Auth Token: fake_token\n"
"Header X-Upload-Content-Length: 17\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"", {{"Location", "https://custom/upload/location"}}),
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
"Header Content-Range: bytes 0-16/17\n"
+ "Timeouts: 5 1 30\n"
"Put body: content1,content2\n",
"", errors::NotFound("important HTTP error 410"),
410),
@@ -515,11 +553,13 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) {
"uploadType=resumable&name=path%2Fwriteable.txt\n"
"Auth Token: fake_token\n"
"Header X-Upload-Content-Length: 17\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"", {{"Location", "https://custom/upload/location"}}),
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
"Header Content-Range: bytes 0-16/17\n"
+ "Timeouts: 5 1 30\n"
"Put body: content1,content2\n",
"")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -529,7 +569,7 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
std::unique_ptr<WritableFile> file;
TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file));
@@ -558,7 +598,7 @@ TEST(GcsFileSystemTest, NewWritableFile_NoObjectName) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
std::unique_ptr<WritableFile> file;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -570,33 +610,38 @@ TEST(GcsFileSystemTest, NewAppendableFile) {
{new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
"Auth Token: fake_token\n"
- "Range: 0-31\n",
+ "Range: 0-31\n"
+ "Timeouts: 5 1 20\n",
"content1,"),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
"uploadType=resumable&name=path%2Fappendable\n"
"Auth Token: fake_token\n"
"Header X-Upload-Content-Length: 17\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"", {{"Location", "https://custom/upload/location"}}),
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
"Header Content-Range: bytes 0-16/17\n"
+ "Timeouts: 5 1 30\n"
"Put body: content1,content2\n",
""),
new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
"Auth Token: fake_token\n"
- "Range: 0-31\n",
+ "Range: 0-31\n"
+ "Timeouts: 5 1 20\n",
"01234567")});
- GcsFileSystem fs(
- std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 32 /* block size */, 32 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */, 0 /* initial retry delay */);
+ GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ 32 /* block size */, 32 /* max bytes */,
+ 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */,
+ 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */,
+ 0 /* initial retry delay */, kTestTimeoutConfig);
// Create an appendable file. This should read the file from GCS, and pull its
// contents into the block cache.
@@ -629,7 +674,7 @@ TEST(GcsFileSystemTest, NewAppendableFile_NoObjectName) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
std::unique_ptr<WritableFile> file;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -642,7 +687,8 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) {
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Frandom_access.txt?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"", content.size(),
"\", \"updated\": \"2016-04-29T23:15:24.896Z\"}")),
new FakeHttpRequest(
@@ -650,7 +696,7 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) {
"path%2Frandom_access.txt\n"
"Auth Token: fake_token\n"
"Range: 0-",
- content.size() - 1, "\n"),
+ content.size() - 1, "\n", "Timeouts: 5 1 20\n"),
content)});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -659,7 +705,7 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
std::unique_ptr<ReadOnlyMemoryRegion> region;
TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile(
@@ -678,7 +724,7 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile_NoObjectName) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
std::unique_ptr<ReadOnlyMemoryRegion> region;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -689,7 +735,8 @@ TEST(GcsFileSystemTest, FileExists_YesAsObject) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Ffile1.txt?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"1010\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -699,7 +746,7 @@ TEST(GcsFileSystemTest, FileExists_YesAsObject) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt"));
}
@@ -709,13 +756,15 @@ TEST(GcsFileSystemTest, FileExists_YesAsFolder) {
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Fsubfolder?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubfolder%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/subfolder/\" }]}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -725,7 +774,7 @@ TEST(GcsFileSystemTest, FileExists_YesAsFolder) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder"));
}
@@ -734,11 +783,13 @@ TEST(GcsFileSystemTest, FileExists_YesAsBucket) {
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"size\": \"100\"}"),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"size\": \"100\"}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -747,7 +798,7 @@ TEST(GcsFileSystemTest, FileExists_YesAsBucket) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
TF_EXPECT_OK(fs.FileExists("gs://bucket1"));
TF_EXPECT_OK(fs.FileExists("gs://bucket1/"));
@@ -758,13 +809,15 @@ TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) {
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Ffile1.txt?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2Ffile1.txt%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": []}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -773,7 +826,7 @@ TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
EXPECT_EQ(errors::Code::NOT_FOUND,
fs.FileExists("gs://bucket/path/file1.txt").code());
@@ -783,11 +836,13 @@ TEST(GcsFileSystemTest, FileExists_NotAsBucket) {
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket2\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket2\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404)});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -796,7 +851,7 @@ TEST(GcsFileSystemTest, FileExists_NotAsBucket) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
fs.FileExists("gs://bucket2/").code());
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -808,29 +863,33 @@ TEST(GcsFileSystemTest, FileExists_StatCache) {
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Ffile1.txt?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"1010\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Fsubfolder?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubfolder%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/subfolder/\" }]}")});
- GcsFileSystem fs(
- std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 3600 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */, 0 /* initial retry delay */);
+ GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 3600 /* stat cache max age */,
+ 0 /* stat cache max entries */,
+ 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */,
+ 0 /* initial retry delay */, kTestTimeoutConfig);
// The stat cache will ensure that repeated lookups don't trigger additional
// HTTP requests.
@@ -845,7 +904,8 @@ TEST(GcsFileSystemTest, GetChildren_NoItems) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
"path%2F\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"prefixes\": [\"path/subpath/\"]}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -854,7 +914,7 @@ TEST(GcsFileSystemTest, GetChildren_NoItems) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
@@ -867,7 +927,8 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
"path%2F\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/file1.txt\" },"
" { \"name\": \"path/file3.txt\" }],"
@@ -879,7 +940,7 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
@@ -893,7 +954,8 @@ TEST(GcsFileSystemTest, GetChildren_SelfDirectoryMarker) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
"path%2F\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/\" },"
" { \"name\": \"path/file3.txt\" }],"
@@ -905,7 +967,7 @@ TEST(GcsFileSystemTest, GetChildren_SelfDirectoryMarker) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay */, kTestTimeoutConfig);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
@@ -918,7 +980,8 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles_NoSlash) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
"path%2F\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/file1.txt\" },"
" { \"name\": \"path/file3.txt\" }],"
@@ -930,7 +993,7 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles_NoSlash) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children));
@@ -943,7 +1006,8 @@ TEST(GcsFileSystemTest, GetChildren_Root) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket-a-b-c/o?"
"fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -952,7 +1016,7 @@ TEST(GcsFileSystemTest, GetChildren_Root) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket-a-b-c", &children));
@@ -965,7 +1029,8 @@ TEST(GcsFileSystemTest, GetChildren_Empty) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix="
"path%2F\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -974,7 +1039,7 @@ TEST(GcsFileSystemTest, GetChildren_Empty) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
@@ -988,7 +1053,8 @@ TEST(GcsFileSystemTest, GetChildren_Pagination) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&"
"prefix=path%2F\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"nextPageToken\": \"ABCD==\", "
"\"items\": [ "
" { \"name\": \"path/file1.txt\" },"
@@ -999,7 +1065,8 @@ TEST(GcsFileSystemTest, GetChildren_Pagination) {
"fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&"
"prefix=path%2F"
"&pageToken=ABCD==\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/file4.txt\" },"
" { \"name\": \"path/file5.txt\" }]}")});
@@ -1011,7 +1078,7 @@ TEST(GcsFileSystemTest, GetChildren_Pagination) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children));
@@ -1025,7 +1092,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_NoWildcard) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/subpath/file2.txt\" }]}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -1035,7 +1103,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_NoWildcard) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
std::vector<string> result;
TF_EXPECT_OK(
@@ -1048,7 +1116,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_BucketAndWildcard) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/file1.txt\" },"
" { \"name\": \"path/subpath/file2.txt\" },"
@@ -1060,7 +1129,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_BucketAndWildcard) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
std::vector<string> result;
TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", &result));
@@ -1074,7 +1143,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_Matches) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/file1.txt\" },"
" { \"name\": \"path/subpath/file2.txt\" },"
@@ -1086,7 +1156,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_Matches) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
std::vector<string> result;
TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file2.txt", &result));
@@ -1098,7 +1168,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_SelfDirectoryMarker) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/\" },"
" { \"name\": \"path/file3.txt\" }]}")});
@@ -1109,7 +1180,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_SelfDirectoryMarker) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
std::vector<string> result;
TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", &result));
@@ -1120,7 +1191,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/file1.txt\" },"
" { \"name\": \"path/subpath/file2.txt\" },"
@@ -1132,7 +1204,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
std::vector<string> result;
TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file3.txt", &result));
@@ -1148,7 +1220,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_OnlyWildcard) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
std::vector<string> result;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -1160,13 +1232,15 @@ TEST(GcsFileSystemTest, GetMatchingPaths_Cache) {
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/subpath/file2.txt\" }]}"),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/file1.txt\" },"
" { \"name\": \"path/subpath/file2.txt\" },"
@@ -1178,7 +1252,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_Cache) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
3600 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
// Repeated calls to fs.GetMatchingPaths on these patterns should not lead to
// any additional HTTP requests to GCS.
@@ -1201,26 +1275,30 @@ TEST(GcsFileSystemTest, DeleteFile) {
{new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n"
"Auth Token: fake_token\n"
- "Range: 0-15\n",
+ "Range: 0-15\n"
+ "Timeouts: 5 1 20\n",
"01234567"),
new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
"/bucket/o/path%2Ffile1.txt\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
""),
new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n"
"Auth Token: fake_token\n"
- "Range: 0-15\n",
+ "Range: 0-15\n"
+ "Timeouts: 5 1 20\n",
"76543210")});
- GcsFileSystem fs(
- std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 16 /* block size */, 16 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */, 0 /* initial retry delay */);
+ GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ 16 /* block size */, 16 /* max bytes */,
+ 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */,
+ 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */,
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
// Do an initial read of the file to load its contents into the block cache.
char scratch[100];
@@ -1246,7 +1324,7 @@ TEST(GcsFileSystemTest, DeleteFile_NoObjectName) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
fs.DeleteFile("gs://bucket/").code());
@@ -1256,7 +1334,8 @@ TEST(GcsFileSystemTest, DeleteDir_Empty) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -1265,7 +1344,7 @@ TEST(GcsFileSystemTest, DeleteDir_Empty) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/"));
}
@@ -1275,12 +1354,14 @@ TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) {
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/\" }]}"),
new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
"/bucket/o/path%2F\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
"")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -1290,7 +1371,7 @@ TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/"));
}
@@ -1298,7 +1379,8 @@ TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) {
TEST(GcsFileSystemTest, DeleteDir_BucketOnly) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?fields=items%2F"
- "name%2CnextPageToken&maxResults=2\nAuth Token: fake_token\n",
+ "name%2CnextPageToken&maxResults=2\nAuth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -1307,7 +1389,7 @@ TEST(GcsFileSystemTest, DeleteDir_BucketOnly) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
TF_EXPECT_OK(fs.DeleteDir("gs://bucket"));
}
@@ -1316,7 +1398,8 @@ TEST(GcsFileSystemTest, DeleteDir_NonEmpty) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/file1.txt\" }]}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -1326,7 +1409,7 @@ TEST(GcsFileSystemTest, DeleteDir_NonEmpty) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
EXPECT_EQ(error::Code::FAILED_PRECONDITION,
fs.DeleteDir("gs://bucket/path/").code());
@@ -1336,7 +1419,8 @@ TEST(GcsFileSystemTest, GetFileSize) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"file.txt?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"1010\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -1346,7 +1430,7 @@ TEST(GcsFileSystemTest, GetFileSize) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
uint64 size;
TF_EXPECT_OK(fs.GetFileSize("gs://bucket/file.txt", &size));
@@ -1362,7 +1446,7 @@ TEST(GcsFileSystemTest, GetFileSize_NoObjectName) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
uint64 size;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -1376,14 +1460,16 @@ TEST(GcsFileSystemTest, RenameFile_Folder) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path1%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path1/subfolder/file1.txt\" }]}"),
// Requesting the full list of files in the folder.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path1%2F\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path1/\" }," // A directory marker.
" { \"name\": \"path1/subfolder/file1.txt\" },"
@@ -1393,13 +1479,15 @@ TEST(GcsFileSystemTest, RenameFile_Folder) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path1%2F/rewriteTo/b/bucket/o/path2%2F\n"
"Auth Token: fake_token\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"{\"done\": true}"),
// Deleting the original directory marker.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path1%2F\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
""),
// Copying the first file.
@@ -1408,13 +1496,15 @@ TEST(GcsFileSystemTest, RenameFile_Folder) {
"path1%2Fsubfolder%2Ffile1.txt/rewriteTo/b/bucket/o/"
"path2%2Fsubfolder%2Ffile1.txt\n"
"Auth Token: fake_token\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"{\"done\": true}"),
// Deleting the first original file.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path1%2Fsubfolder%2Ffile1.txt\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
""),
// Copying the second file.
@@ -1422,13 +1512,15 @@ TEST(GcsFileSystemTest, RenameFile_Folder) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path1%2Ffile2.txt/rewriteTo/b/bucket/o/path2%2Ffile2.txt\n"
"Auth Token: fake_token\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"{\"done\": true}"),
// Deleting the second original file.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path1%2Ffile2.txt\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
"")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -1438,7 +1530,7 @@ TEST(GcsFileSystemTest, RenameFile_Folder) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
TF_EXPECT_OK(fs.RenameFile("gs://bucket/path1", "gs://bucket/path2/"));
}
@@ -1448,25 +1540,29 @@ TEST(GcsFileSystemTest, RenameFile_Object) {
{new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n"
"Auth Token: fake_token\n"
- "Range: 0-15\n",
+ "Range: 0-15\n"
+ "Timeouts: 5 1 20\n",
"01234567"),
new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n"
"Auth Token: fake_token\n"
- "Range: 0-15\n",
+ "Range: 0-15\n"
+ "Timeouts: 5 1 20\n",
"76543210"),
// IsDirectory is checking whether there are children objects.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{}"),
// IsDirectory is checking if the path exists as an object.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Fsrc.txt?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"1010\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
// Copying to the new location.
@@ -1474,33 +1570,38 @@ TEST(GcsFileSystemTest, RenameFile_Object) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
"Auth Token: fake_token\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"{\"done\": true}"),
// Deleting the original file.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Fsrc.txt\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
""),
new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n"
"Auth Token: fake_token\n"
- "Range: 0-15\n",
+ "Range: 0-15\n"
+ "Timeouts: 5 1 20\n",
"89abcdef"),
new FakeHttpRequest(
"Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n"
"Auth Token: fake_token\n"
- "Range: 0-15\n",
+ "Range: 0-15\n"
+ "Timeouts: 5 1 20\n",
"fedcba98")});
- GcsFileSystem fs(
- std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 16 /* block size */, 64 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */, 0 /* initial retry delay */);
+ GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ 16 /* block size */, 64 /* max bytes */,
+ 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */,
+ 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */,
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
// Do an initial read of the source and destination files to load their
// contents into the block cache.
char scratch[100];
@@ -1531,13 +1632,15 @@ TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{}"),
// IsDirectory is checking if the path exists as an object.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Fsrc.txt?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"1010\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
// Copying to the new location.
@@ -1545,13 +1648,15 @@ TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
"Auth Token: fake_token\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"{\"done\": true}"),
// Deleting the original file - the deletion returns a failure.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Fsrc.txt\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
"", errors::Unavailable("503"), 503),
// Deleting the original file again - the deletion returns NOT_FOUND.
@@ -1559,6 +1664,7 @@ TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Fsrc.txt\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
"", errors::NotFound("404"), 404)});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -1568,7 +1674,7 @@ TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
TF_EXPECT_OK(
fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt"));
@@ -1582,13 +1688,15 @@ TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{}"),
// IsDirectory is checking if the path exists as an object.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Fsrc.txt?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"1010\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
// Copying to the new location.
@@ -1596,7 +1704,8 @@ TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n"
"Auth Token: fake_token\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"{\"done\": false}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -1605,7 +1714,7 @@ TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
EXPECT_EQ(
errors::Code::UNIMPLEMENTED,
@@ -1617,7 +1726,8 @@ TEST(GcsFileSystemTest, Stat_Object) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"file.txt?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"1010\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -1627,7 +1737,7 @@ TEST(GcsFileSystemTest, Stat_Object) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
FileStatistics stat;
TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat));
@@ -1641,13 +1751,15 @@ TEST(GcsFileSystemTest, Stat_Folder) {
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"subfolder?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"subfolder/\" }]}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -1657,7 +1769,7 @@ TEST(GcsFileSystemTest, Stat_Folder) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
FileStatistics stat;
TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", &stat));
@@ -1671,13 +1783,15 @@ TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) {
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -1686,7 +1800,7 @@ TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
FileStatistics stat;
EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/path", &stat).code());
@@ -1695,7 +1809,8 @@ TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) {
TEST(GcsFileSystemTest, Stat_Bucket) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -1704,7 +1819,7 @@ TEST(GcsFileSystemTest, Stat_Bucket) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
FileStatistics stat;
TF_EXPECT_OK(fs.Stat("gs://bucket/", &stat));
@@ -1716,7 +1831,8 @@ TEST(GcsFileSystemTest, Stat_Bucket) {
TEST(GcsFileSystemTest, Stat_BucketNotFound) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404)});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -1725,7 +1841,7 @@ TEST(GcsFileSystemTest, Stat_BucketNotFound) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
FileStatistics stat;
EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/", &stat).code());
@@ -1736,29 +1852,33 @@ TEST(GcsFileSystemTest, Stat_Cache) {
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"file.txt?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"1010\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}")),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"subfolder?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"subfolder/\" }]}")});
- GcsFileSystem fs(
- std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 3600 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */, 0 /* initial retry delay */);
+ GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 3600 /* stat cache max age */,
+ 0 /* stat cache max entries */,
+ 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */,
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
// Repeated calls to fs.Stat on these paths should not lead to any additional
// HTTP requests to GCS.
@@ -1781,12 +1901,14 @@ TEST(GcsFileSystemTest, IsDirectory_NotFound) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=file.txt%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{}"),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"file.txt?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404)});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -1795,7 +1917,7 @@ TEST(GcsFileSystemTest, IsDirectory_NotFound) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
EXPECT_EQ(error::Code::NOT_FOUND,
fs.IsDirectory("gs://bucket/file.txt").code());
@@ -1807,12 +1929,14 @@ TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=file.txt%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{}"),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"file.txt?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"1010\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -1822,7 +1946,7 @@ TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
EXPECT_EQ(error::Code::FAILED_PRECONDITION,
fs.IsDirectory("gs://bucket/file.txt").code());
@@ -1834,13 +1958,15 @@ TEST(GcsFileSystemTest, IsDirectory_Yes) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [{\"name\": \"subfolder/\"}]}"),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [{\"name\": \"subfolder/\"}]}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -1849,7 +1975,7 @@ TEST(GcsFileSystemTest, IsDirectory_Yes) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder"));
TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder/"));
@@ -1859,11 +1985,13 @@ TEST(GcsFileSystemTest, IsDirectory_Bucket) {
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{}"),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{}")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -1872,7 +2000,7 @@ TEST(GcsFileSystemTest, IsDirectory_Bucket) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
TF_EXPECT_OK(fs.IsDirectory("gs://bucket"));
TF_EXPECT_OK(fs.IsDirectory("gs://bucket/"));
@@ -1881,7 +2009,8 @@ TEST(GcsFileSystemTest, IsDirectory_Bucket) {
TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) {
std::vector<HttpRequest*> requests({new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404)});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -1890,7 +2019,7 @@ TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/").code());
}
@@ -1902,10 +2031,12 @@ TEST(GcsFileSystemTest, CreateDir_Folder) {
"uploadType=resumable&name=subpath%2F\n"
"Auth Token: fake_token\n"
"Header X-Upload-Content-Length: 0\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"", {{"Location", "https://custom/upload/location"}}),
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 30\n"
"Put body: \n",
""),
new FakeHttpRequest(
@@ -1913,10 +2044,12 @@ TEST(GcsFileSystemTest, CreateDir_Folder) {
"uploadType=resumable&name=subpath%2F\n"
"Auth Token: fake_token\n"
"Header X-Upload-Content-Length: 0\n"
- "Post: yes\n",
+ "Post: yes\n"
+ "Timeouts: 5 1 10\n",
"", {{"Location", "https://custom/upload/location"}}),
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 30\n"
"Put body: \n",
"")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -1926,7 +2059,7 @@ TEST(GcsFileSystemTest, CreateDir_Folder) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath"));
TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath/"));
@@ -1936,11 +2069,13 @@ TEST(GcsFileSystemTest, CreateDir_Bucket) {
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
""),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -1949,7 +2084,7 @@ TEST(GcsFileSystemTest, CreateDir_Bucket) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
TF_EXPECT_OK(fs.CreateDir("gs://bucket/"));
TF_EXPECT_OK(fs.CreateDir("gs://bucket"));
@@ -1962,14 +2097,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_Ok) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/file1.txt\" }]}"),
// GetChildren recursively.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/\" }," // The current directory's marker.
" { \"name\": \"path/file1.txt\" },"
@@ -1979,30 +2116,35 @@ TEST(GcsFileSystemTest, DeleteRecursively_Ok) {
new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
"/bucket/o/path%2F\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
""),
// Delete the object - fails and will be retried.
new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
"/bucket/o/path%2Ffile1.txt\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
"", errors::Unavailable("500"), 500),
// Delete the object again.
new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
"/bucket/o/path%2Ffile1.txt\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
""),
// Delete the object.
new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
"/bucket/o/path%2Fsubpath%2Ffile2.txt\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
""),
// Delete the object.
new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
"/bucket/o/path%2Ffile3.txt\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
"")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -2012,7 +2154,7 @@ TEST(GcsFileSystemTest, DeleteRecursively_Ok) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
int64 undeleted_files, undeleted_dirs;
TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files,
@@ -2028,14 +2170,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/file1.txt\" }]}"),
// Calling GetChildren recursively.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2F\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/file1.txt\" },"
" { \"name\": \"path/subpath/\" },"
@@ -2045,12 +2189,14 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) {
new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
"/bucket/o/path%2Ffile1.txt\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
""),
// Deleting the directory marker gs://bucket/path/ - fails with 404.
new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
"/bucket/o/path%2Fsubpath%2F\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
"", errors::NotFound("404"), 404),
// Checking if gs://bucket/path/subpath/ is a folder - it is.
@@ -2058,19 +2204,22 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
strings::StrCat("{\"items\": [ "
" { \"name\": \"path/subpath/\" }]}")),
// Deleting the object gs://bucket/path/subpath/file2.txt
new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
"/bucket/o/path%2Fsubpath%2Ffile2.txt\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
""),
// Deleting the object s://bucket/path/file3.txt - fails with 404.
new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
"/bucket/o/path%2Ffile3.txt\n"
"Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n"
"Delete: yes\n",
"", errors::NotFound("404"), 404),
// Checking if gs://bucket/path/file3.txt/ is a folder - it's not.
@@ -2078,13 +2227,15 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2Ffile3.txt%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{}"),
// Checking if gs://bucket/path/file3.txt is an object - fails with 404.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path%2Ffile3.txt?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404)});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
@@ -2094,7 +2245,7 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
int64 undeleted_files, undeleted_dirs;
TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files,
@@ -2110,13 +2261,15 @@ TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) {
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2F"
"&maxResults=1\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"{}"),
// IsDirectory is checking if the path exists as an object.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o/"
"path?fields=size%2Cupdated\n"
- "Auth Token: fake_token\n",
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404)});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
@@ -2125,7 +2278,7 @@ TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) {
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */,
- 0 /* initial retry delay */);
+ 0 /* initial retry delay*/, kTestTimeoutConfig);
int64 undeleted_files, undeleted_dirs;
EXPECT_EQ(error::Code::NOT_FOUND,
@@ -2142,6 +2295,11 @@ TEST(GcsFileSystemTest, OverrideCacheParameters) {
EXPECT_EQ(128 * 1024 * 1024, fs1.block_size());
EXPECT_EQ(2 * fs1.block_size(), fs1.max_bytes());
EXPECT_EQ(0, fs1.max_staleness());
+ EXPECT_EQ(120, fs1.timeouts().connect);
+ EXPECT_EQ(60, fs1.timeouts().idle);
+ EXPECT_EQ(3600, fs1.timeouts().metadata);
+ EXPECT_EQ(3600, fs1.timeouts().read);
+ EXPECT_EQ(3600, fs1.timeouts().write);
// Verify legacy readahead buffer override sets block size.
setenv("GCS_READAHEAD_BUFFER_SIZE_BYTES", "123456789", 1);
@@ -2167,6 +2325,19 @@ TEST(GcsFileSystemTest, OverrideCacheParameters) {
EXPECT_EQ(32, fs4.stat_cache_max_entries());
EXPECT_EQ(30, fs4.matching_paths_cache_max_age());
EXPECT_EQ(64, fs4.matching_paths_cache_max_entries());
+
+ // Verify timeout overrides.
+ setenv("GCS_REQUEST_CONNECTION_TIMEOUT_SECS", "10", 1);
+ setenv("GCS_REQUEST_IDLE_TIMEOUT_SECS", "5", 1);
+ setenv("GCS_METADATA_REQUEST_TIMEOUT_SECS", "20", 1);
+ setenv("GCS_READ_REQUEST_TIMEOUT_SECS", "30", 1);
+ setenv("GCS_WRITE_REQUEST_TIMEOUT_SECS", "40", 1);
+ GcsFileSystem fs5;
+ EXPECT_EQ(10, fs5.timeouts().connect);
+ EXPECT_EQ(5, fs5.timeouts().idle);
+ EXPECT_EQ(20, fs5.timeouts().metadata);
+ EXPECT_EQ(30, fs5.timeouts().read);
+ EXPECT_EQ(40, fs5.timeouts().write);
}
} // namespace
diff --git a/tensorflow/core/platform/cloud/http_request.h b/tensorflow/core/platform/cloud/http_request.h
index 02d9e9054a..95a436c622 100644
--- a/tensorflow/core/platform/cloud/http_request.h
+++ b/tensorflow/core/platform/cloud/http_request.h
@@ -118,6 +118,16 @@ class HttpRequest {
// Url encodes str and returns a new string.
virtual string EscapeString(const string& str) = 0;
+ /// \brief Set timeouts for this request.
+ ///
+ /// The connection parameter controls how long we should wait for the
+ /// connection to be established. The inactivity parameter controls how long
+ /// we should wait between additional responses from the server. Finally the
+ /// total parameter controls the maximum total connection time to prevent
+ /// hanging indefinitely.
+ virtual Status SetTimeouts(uint32 connection, uint32 inactivity,
+ uint32 total) = 0;
+
TF_DISALLOW_COPY_AND_ASSIGN(HttpRequest);
};
diff --git a/tensorflow/core/platform/cloud/http_request_fake.h b/tensorflow/core/platform/cloud/http_request_fake.h
index bfe04f6363..22398d739c 100644
--- a/tensorflow/core/platform/cloud/http_request_fake.h
+++ b/tensorflow/core/platform/cloud/http_request_fake.h
@@ -37,13 +37,14 @@ class FakeHttpRequest : public CurlHttpRequest {
public:
/// Return the response for the given request.
FakeHttpRequest(const string& request, const string& response)
- : FakeHttpRequest(request, response, Status::OK(), nullptr, {}, 200) {}
+ : FakeHttpRequest(request, response, Status::OK(), nullptr, {}, 200, {}) {
+ }
/// Return the response with headers for the given request.
FakeHttpRequest(const string& request, const string& response,
const std::map<string, string>& response_headers)
: FakeHttpRequest(request, response, Status::OK(), nullptr,
- response_headers, 200) {}
+ response_headers, 200, {}) {}
/// \brief Return the response for the request and capture the POST body.
///
@@ -51,13 +52,13 @@ class FakeHttpRequest : public CurlHttpRequest {
FakeHttpRequest(const string& request, const string& response,
string* captured_post_body)
: FakeHttpRequest(request, response, Status::OK(), captured_post_body, {},
- 200) {}
+ 200, {}) {}
/// \brief Return the response and the status for the given request.
FakeHttpRequest(const string& request, const string& response,
Status response_status, uint64 response_code)
: FakeHttpRequest(request, response, response_status, nullptr, {},
- response_code) {}
+ response_code, {}) {}
/// \brief Return the response and the status for the given request
/// and capture the POST body.
@@ -66,13 +67,15 @@ class FakeHttpRequest : public CurlHttpRequest {
FakeHttpRequest(const string& request, const string& response,
Status response_status, string* captured_post_body,
const std::map<string, string>& response_headers,
- uint64 response_code)
+ uint64 response_code,
+ absl::optional<std::tuple<uint32, uint32, uint32>> timeouts)
: expected_request_(request),
response_(response),
response_status_(response_status),
captured_post_body_(captured_post_body),
response_headers_(response_headers),
- response_code_(response_code) {}
+ response_code_(response_code),
+ timeouts_(timeouts) {}
Status Init() override { return Status::OK(); }
Status SetUri(const string& uri) override {
@@ -160,6 +163,13 @@ class FakeHttpRequest : public CurlHttpRequest {
virtual uint64 GetResponseCode() const override { return response_code_; }
+ Status SetTimeouts(uint32 connection, uint32 inactivity,
+ uint32 total) override {
+ actual_request_ += strings::StrCat("Timeouts: ", connection, " ",
+ inactivity, " ", total, "\n");
+ return Status::OK();
+ }
+
private:
std::vector<char>* buffer_ = nullptr;
string expected_request_;
@@ -169,6 +179,7 @@ class FakeHttpRequest : public CurlHttpRequest {
string* captured_post_body_ = nullptr;
std::map<string, string> response_headers_;
uint64 response_code_ = 0;
+ absl::optional<std::tuple<uint32, uint32, uint32>> timeouts_;
};
/// Fake HttpRequest factory for testing.