From d8e98c8753e3d89e6d4e989e52e910495c4f64ac Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Fri, 1 Dec 2017 11:08:51 -0800 Subject: Customize request timeouts for GCS filesystem. PiperOrigin-RevId: 177614538 --- .../core/platform/cloud/curl_http_request.cc | 29 +- tensorflow/core/platform/cloud/curl_http_request.h | 12 + .../core/platform/cloud/gcs_dns_cache_test.cc | 5 + tensorflow/core/platform/cloud/gcs_file_system.cc | 65 ++- tensorflow/core/platform/cloud/gcs_file_system.h | 42 +- .../core/platform/cloud/gcs_file_system_test.cc | 649 +++++++++++++-------- tensorflow/core/platform/cloud/http_request.h | 10 + tensorflow/core/platform/cloud/http_request_fake.h | 23 +- 8 files changed, 572 insertions(+), 263 deletions(-) diff --git a/tensorflow/core/platform/cloud/curl_http_request.cc b/tensorflow/core/platform/cloud/curl_http_request.cc index 4581a0870a..f7fbfe971e 100644 --- a/tensorflow/core/platform/cloud/curl_http_request.cc +++ b/tensorflow/core/platform/cloud/curl_http_request.cc @@ -29,16 +29,6 @@ namespace { // Set to 1 to enable verbose debug output from curl. constexpr uint64 kVerboseOutput = 0; -// Timeout for the whole request. Set only to prevent hanging indefinitely. -constexpr uint32 kRequestTimeoutSeconds = 3600; // 1 hour - -// Timeout for the connection phase. -constexpr uint32 kConnectTimeoutSeconds = 120; // 2 minutes - -// The maximum period of request inactivity, after which the request -// is terminated. -constexpr uint64 kInactivityTimeoutSeconds = 60; // 1 minute - // Proxy to the real libcurl implementation. class LibCurlProxy : public LibCurl { public: @@ -161,9 +151,6 @@ Status CurlHttpRequest::Init() { strings::StrCat("TensorFlow/", TF_VERSION_STRING).c_str()); // Do not use signals for timeouts - does not work in multi-threaded programs. libcurl_->curl_easy_setopt(curl_, CURLOPT_NOSIGNAL, 1L); - libcurl_->curl_easy_setopt(curl_, CURLOPT_TIMEOUT, kRequestTimeoutSeconds); - libcurl_->curl_easy_setopt(curl_, CURLOPT_CONNECTTIMEOUT, - kConnectTimeoutSeconds); libcurl_->curl_easy_setopt(curl_, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); @@ -336,6 +323,16 @@ Status CurlHttpRequest::SetResultBuffer(std::vector* out_buffer) { return Status::OK(); } +Status CurlHttpRequest::SetTimeouts(uint32 connection, uint32 inactivity, + uint32 total) { + TF_RETURN_IF_ERROR(CheckInitialized()); + TF_RETURN_IF_ERROR(CheckNotSent()); + connect_timeout_secs_ = connection; + inactivity_timeout_secs_ = inactivity; + request_timeout_secs_ = total; + return Status::OK(); +} + size_t CurlHttpRequest::WriteCallback(const void* ptr, size_t size, size_t nmemb, void* this_object) { CHECK(ptr); @@ -399,6 +396,10 @@ Status CurlHttpRequest::Send() { libcurl_->curl_easy_setopt(curl_, CURLOPT_HEADERFUNCTION, &CurlHttpRequest::HeaderCallback); + libcurl_->curl_easy_setopt(curl_, CURLOPT_TIMEOUT, request_timeout_secs_); + libcurl_->curl_easy_setopt(curl_, CURLOPT_CONNECTTIMEOUT, + connect_timeout_secs_); + char error_buffer[CURL_ERROR_SIZE] = {0}; libcurl_->curl_easy_setopt(curl_, CURLOPT_ERRORBUFFER, error_buffer); @@ -529,7 +530,7 @@ int CurlHttpRequest::ProgressCallback(void* this_object, curl_off_t dltotal, return 0; } - if (now - that->last_progress_timestamp_ > kInactivityTimeoutSeconds) { + if (now - that->last_progress_timestamp_ > that->inactivity_timeout_secs_) { LOG(ERROR) << "The transmission of request " << this_object << " (URI: " << that->uri_ << ") has been stuck at " << current_progress << " of " << dltotal + ultotal diff --git a/tensorflow/core/platform/cloud/curl_http_request.h b/tensorflow/core/platform/cloud/curl_http_request.h index 9e5ae61016..cf249450df 100644 --- a/tensorflow/core/platform/cloud/curl_http_request.h +++ b/tensorflow/core/platform/cloud/curl_http_request.h @@ -120,6 +120,9 @@ class CurlHttpRequest : public HttpRequest { // Url encodes str and returns a new string. string EscapeString(const string& str) override; + Status SetTimeouts(uint32 connection, uint32 inactivity, + uint32 total) override; + private: /// A write callback in the form which can be accepted by libcurl. static size_t WriteCallback(const void* ptr, size_t size, size_t nmemb, @@ -162,6 +165,15 @@ class CurlHttpRequest : public HttpRequest { // The last progress in terms of bytes transmitted. curl_off_t last_progress_bytes_ = 0; + // The maximum period of request inactivity. + uint32 inactivity_timeout_secs_ = 60; // 1 minute + + // Timeout for the connection phase. + uint32 connect_timeout_secs_ = 120; // 2 minutes + + // Tiemout for the whole request. Set only to prevent hanging indefinitely. + uint32 request_timeout_secs_ = 3600; // 1 hour + // Members to enforce the usage flow. bool is_initialized_ = false; bool is_uri_set_ = false; diff --git a/tensorflow/core/platform/cloud/gcs_dns_cache_test.cc b/tensorflow/core/platform/cloud/gcs_dns_cache_test.cc index 8d1a108f30..266879ddf5 100644 --- a/tensorflow/core/platform/cloud/gcs_dns_cache_test.cc +++ b/tensorflow/core/platform/cloud/gcs_dns_cache_test.cc @@ -64,6 +64,11 @@ class TestHttpRequest : public HttpRequest { Status Send() override { return Status::OK(); } string EscapeString(const string& str) override { return ""; } + Status SetTimeouts(uint32 connection, uint32 inactivity, + uint32 total) override { + return Status::OK(); + } + std::map resolve_overrides_; }; diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc index 45e9b05092..ab82643ad5 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system.cc @@ -94,6 +94,20 @@ const FileStatistics DIRECTORY_STAT(0, 0, true); // variable to a positive integer describing the frequency used to refresh the // userspace DNS cache. constexpr char kResolveCacheSecs[] = "GCS_RESOLVE_REFRESH_SECS"; +// The environment variable to configure the http request's connection timeout. +constexpr char kRequestConnectionTimeout[] = + "GCS_REQUEST_CONNECTION_TIMEOUT_SECS"; +// The environment varaible to configure the http request's idle timeout. +constexpr char kRequestIdleTimeout[] = "GCS_REQUEST_IDLE_TIMEOUT_SECS"; +// The environment variable to configure the overall request timeout for +// metadata requests. +constexpr char kMetadataRequestTimeout[] = "GCS_METADATA_REQUEST_TIMEOUT_SECS"; +// The environment variable to configure the overall request timeout for +// block reads requests. +constexpr char kReadRequestTimeout[] = "GCS_READ_REQUEST_TIMEOUT_SECS"; +// The environment variable to configure the overall request timeout for +// upload requests. +constexpr char kWriteRequestTimeout[] = "GCS_WRITE_REQUEST_TIMEOUT_SECS"; Status GetTmpFilename(string* filename) { if (!filename) { @@ -283,12 +297,14 @@ class GcsWritableFile : public WritableFile { GcsWritableFile(const string& bucket, const string& object, AuthProvider* auth_provider, HttpRequest::Factory* http_request_factory, + GcsFileSystem::TimeoutConfig* timeouts, std::function file_cache_erase, int64 initial_retry_delay_usec) : bucket_(bucket), object_(object), auth_provider_(auth_provider), http_request_factory_(http_request_factory), + timeouts_(timeouts), file_cache_erase_(std::move(file_cache_erase)), sync_needed_(true), initial_retry_delay_usec_(initial_retry_delay_usec) { @@ -307,12 +323,14 @@ class GcsWritableFile : public WritableFile { AuthProvider* auth_provider, const string& tmp_content_filename, HttpRequest::Factory* http_request_factory, + GcsFileSystem::TimeoutConfig* timeouts, std::function file_cache_erase, int64 initial_retry_delay_usec) : bucket_(bucket), object_(object), auth_provider_(auth_provider), http_request_factory_(http_request_factory), + timeouts_(timeouts), file_cache_erase_(std::move(file_cache_erase)), sync_needed_(true), initial_retry_delay_usec_(initial_retry_delay_usec) { @@ -446,6 +464,8 @@ class GcsWritableFile : public WritableFile { std::to_string(file_size))); TF_RETURN_IF_ERROR(request->SetPostEmptyBody()); TF_RETURN_IF_ERROR(request->SetResultBuffer(&output_buffer)); + TF_RETURN_IF_ERROR(request->SetTimeouts(timeouts_->connect, timeouts_->idle, + timeouts_->metadata)); TF_RETURN_WITH_CONTEXT_IF_ERROR( request->Send(), " when initiating an upload to ", GetGcsPath()); *session_uri = request->GetResponseHeader("Location"); @@ -477,6 +497,8 @@ class GcsWritableFile : public WritableFile { TF_RETURN_IF_ERROR(request->Init()); TF_RETURN_IF_ERROR(request->SetUri(session_uri)); TF_RETURN_IF_ERROR(request->AddAuthBearerHeader(auth_token)); + TF_RETURN_IF_ERROR(request->SetTimeouts(timeouts_->connect, timeouts_->idle, + timeouts_->metadata)); TF_RETURN_IF_ERROR(request->AddHeader( "Content-Range", strings::StrCat("bytes */", file_size))); TF_RETURN_IF_ERROR(request->SetPutEmptyBody()); @@ -531,6 +553,9 @@ class GcsWritableFile : public WritableFile { "Content-Range", strings::StrCat("bytes ", start_offset, "-", file_size - 1, "/", file_size))); } + TF_RETURN_IF_ERROR(request->SetTimeouts(timeouts_->connect, timeouts_->idle, + timeouts_->write)); + TF_RETURN_IF_ERROR( request->SetPutFromFile(tmp_content_filename_, start_offset)); TF_RETURN_WITH_CONTEXT_IF_ERROR(request->Send(), " when uploading ", @@ -550,6 +575,7 @@ class GcsWritableFile : public WritableFile { string tmp_content_filename_; std::ofstream outfile_; HttpRequest::Factory* http_request_factory_; + GcsFileSystem::TimeoutConfig* timeouts_; std::function file_cache_erase_; bool sync_needed_; // whether there is buffered data that needs to be synced int64 initial_retry_delay_usec_; @@ -635,6 +661,25 @@ GcsFileSystem::GcsFileSystem() &resolve_frequency_secs)) { dns_cache_.reset(new GcsDnsCache(resolve_frequency_secs)); } + // Apply the overrides for request timeouts + uint32 timeout_value; + if (GetEnvVar(kRequestConnectionTimeout, strings::safe_strtou32, + &timeout_value)) { + timeouts_.connect = timeout_value; + } + if (GetEnvVar(kRequestIdleTimeout, strings::safe_strtou32, &timeout_value)) { + timeouts_.idle = timeout_value; + } + if (GetEnvVar(kMetadataRequestTimeout, strings::safe_strtou32, + &timeout_value)) { + timeouts_.metadata = timeout_value; + } + if (GetEnvVar(kReadRequestTimeout, strings::safe_strtou32, &timeout_value)) { + timeouts_.read = timeout_value; + } + if (GetEnvVar(kWriteRequestTimeout, strings::safe_strtou32, &timeout_value)) { + timeouts_.write = timeout_value; + } } GcsFileSystem::GcsFileSystem( @@ -643,7 +688,8 @@ GcsFileSystem::GcsFileSystem( size_t block_size, size_t max_bytes, uint64 max_staleness, uint64 stat_cache_max_age, size_t stat_cache_max_entries, uint64 matching_paths_cache_max_age, - size_t matching_paths_cache_max_entries, int64 initial_retry_delay_usec) + size_t matching_paths_cache_max_entries, int64 initial_retry_delay_usec, + TimeoutConfig timeouts) : auth_provider_(std::move(auth_provider)), http_request_factory_(std::move(http_request_factory)), file_block_cache_( @@ -651,6 +697,7 @@ GcsFileSystem::GcsFileSystem( stat_cache_(new StatCache(stat_cache_max_age, stat_cache_max_entries)), matching_paths_cache_(new MatchingPathsCache( matching_paths_cache_max_age, matching_paths_cache_max_entries)), + timeouts_(timeouts), initial_retry_delay_usec_(initial_retry_delay_usec) {} Status GcsFileSystem::NewRandomAccessFile( @@ -689,6 +736,8 @@ Status GcsFileSystem::LoadBufferFromGCS(const string& filename, size_t offset, TF_RETURN_IF_ERROR(request->AddAuthBearerHeader(auth_token)); TF_RETURN_IF_ERROR(request->SetRange(offset, offset + n - 1)); TF_RETURN_IF_ERROR(request->SetResultBuffer(out)); + TF_RETURN_IF_ERROR( + request->SetTimeouts(timeouts_.connect, timeouts_.idle, timeouts_.read)); if (dns_cache_) { TF_RETURN_IF_ERROR(dns_cache_->AnnotateRequest(request.get())); @@ -723,7 +772,7 @@ Status GcsFileSystem::NewWritableFile(const string& fname, TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object)); result->reset(new GcsWritableFile( bucket, object, auth_provider_.get(), http_request_factory_.get(), - [this, fname]() { file_block_cache_->RemoveFile(fname); }, + &timeouts_, [this, fname]() { file_block_cache_->RemoveFile(fname); }, initial_retry_delay_usec_)); return Status::OK(); } @@ -764,7 +813,7 @@ Status GcsFileSystem::NewAppendableFile(const string& fname, TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object)); result->reset(new GcsWritableFile( bucket, object, auth_provider_.get(), old_content_filename, - http_request_factory_.get(), + http_request_factory_.get(), &timeouts_, [this, fname]() { file_block_cache_->RemoveFile(fname); }, initial_retry_delay_usec_)); return Status::OK(); @@ -852,6 +901,8 @@ Status GcsFileSystem::StatForObject(const string& fname, const string& bucket, "?fields=size%2Cupdated"))); TF_RETURN_IF_ERROR(request->AddAuthBearerHeader(auth_token)); TF_RETURN_IF_ERROR(request->SetResultBuffer(&output_buffer)); + TF_RETURN_IF_ERROR(request->SetTimeouts( + timeouts_.connect, timeouts_.idle, timeouts_.metadata)); if (dns_cache_) { TF_RETURN_IF_ERROR(dns_cache_->AnnotateRequest(request.get())); @@ -902,6 +953,8 @@ Status GcsFileSystem::BucketExists(const string& bucket, bool* result) { TF_RETURN_IF_ERROR( request->SetUri(strings::StrCat(kGcsUriBase, "b/", bucket))); TF_RETURN_IF_ERROR(request->AddAuthBearerHeader(auth_token)); + TF_RETURN_IF_ERROR(request->SetTimeouts(timeouts_.connect, timeouts_.idle, + timeouts_.metadata)); const Status status = request->Send(); switch (status.code()) { case errors::Code::OK: @@ -1033,6 +1086,8 @@ Status GcsFileSystem::GetChildrenBounded(const string& dirname, TF_RETURN_IF_ERROR(request->SetUri(uri)); TF_RETURN_IF_ERROR(request->AddAuthBearerHeader(auth_token)); TF_RETURN_IF_ERROR(request->SetResultBuffer(&output_buffer)); + TF_RETURN_IF_ERROR(request->SetTimeouts(timeouts_.connect, timeouts_.idle, + timeouts_.metadata)); if (dns_cache_) { TF_RETURN_IF_ERROR(dns_cache_->AnnotateRequest(request.get())); @@ -1157,6 +1212,8 @@ Status GcsFileSystem::DeleteFile(const string& fname) { TF_RETURN_IF_ERROR(request->SetUri(strings::StrCat( kGcsUriBase, "b/", bucket, "/o/", request->EscapeString(object)))); TF_RETURN_IF_ERROR(request->AddAuthBearerHeader(auth_token)); + TF_RETURN_IF_ERROR(request->SetTimeouts(timeouts_.connect, timeouts_.idle, + timeouts_.metadata)); TF_RETURN_IF_ERROR(request->SetDeleteRequest()); TF_RETURN_WITH_CONTEXT_IF_ERROR(request->Send(), " when deleting ", fname); file_block_cache_->RemoveFile(fname); @@ -1251,6 +1308,8 @@ Status GcsFileSystem::RenameObject(const string& src, const string& target) { request->EscapeString(target_object)))); TF_RETURN_IF_ERROR(request->AddAuthBearerHeader(auth_token)); TF_RETURN_IF_ERROR(request->SetPostEmptyBody()); + TF_RETURN_IF_ERROR(request->SetTimeouts(timeouts_.connect, timeouts_.idle, + timeouts_.metadata)); std::vector output_buffer; TF_RETURN_IF_ERROR(request->SetResultBuffer(&output_buffer)); TF_RETURN_WITH_CONTEXT_IF_ERROR(request->Send(), " when renaming ", src, diff --git a/tensorflow/core/platform/cloud/gcs_file_system.h b/tensorflow/core/platform/cloud/gcs_file_system.h index 4b4853c838..7cfcebd5c9 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.h +++ b/tensorflow/core/platform/cloud/gcs_file_system.h @@ -35,6 +35,8 @@ namespace tensorflow { /// which adds retry logic to GCS operations. class GcsFileSystem : public FileSystem { public: + struct TimeoutConfig; + GcsFileSystem(); GcsFileSystem(std::unique_ptr auth_provider, std::unique_ptr http_request_factory, @@ -42,7 +44,7 @@ class GcsFileSystem : public FileSystem { uint64 stat_cache_max_age, size_t stat_cache_max_entries, uint64 matching_paths_cache_max_age, size_t matching_paths_cache_max_entries, - int64 initial_retry_delay_usec); + int64 initial_retry_delay_usec, TimeoutConfig timeouts); Status NewRandomAccessFile( const string& filename, @@ -87,6 +89,7 @@ class GcsFileSystem : public FileSystem { size_t block_size() const { return file_block_cache_->block_size(); } size_t max_bytes() const { return file_block_cache_->max_bytes(); } uint64 max_staleness() const { return file_block_cache_->max_staleness(); } + TimeoutConfig timeouts() const { return timeouts_; } uint64 stat_cache_max_age() const { return stat_cache_->max_age(); } size_t stat_cache_max_entries() const { return stat_cache_->max_entries(); } @@ -98,6 +101,41 @@ class GcsFileSystem : public FileSystem { return matching_paths_cache_->max_entries(); } + /// Structure containing the information for timeouts related to accessing the + /// GCS APIs. + /// + /// All values are in seconds. + struct TimeoutConfig { + // The request connection timeout. If a connection cannot be established + // within `connect` seconds, abort the request. + uint32 connect = 120; // 2 minutes + + // The request idle timeout. If a request has seen no activity in `idle` + // seconds, abort the request. + uint32 idle = 60; // 1 minute + + // The maximum total time a metadata request can take. If a request has not + // completed within `metadata` seconds, the request is aborted. + uint32 metadata = 3600; // 1 hour + + // The maximum total time a block read request can take. If a request has + // not completed within `read` seconds, the request is aborted. + uint32 read = 3600; // 1 hour + + // The maximum total time an upload request can take. If a request has not + // completed within `write` seconds, the request is aborted. + uint32 write = 3600; // 1 hour + + TimeoutConfig() {} + TimeoutConfig(uint32 connect, uint32 idle, uint32 metadata, uint32 read, + uint32 write) + : connect(connect), + idle(idle), + metadata(metadata), + read(read), + write(write) {} + }; + private: /// \brief Checks if the bucket exists. Returns OK if the check succeeded. /// @@ -150,6 +188,8 @@ class GcsFileSystem : public FileSystem { using MatchingPathsCache = ExpiringLRUCache>; std::unique_ptr matching_paths_cache_; + TimeoutConfig timeouts_; + /// The initial delay for exponential backoffs when retrying failed calls. const int64 initial_retry_delay_usec_ = 1000000L; diff --git a/tensorflow/core/platform/cloud/gcs_file_system_test.cc b/tensorflow/core/platform/cloud/gcs_file_system_test.cc index 7614ec4d7f..01f4fd8688 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system_test.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system_test.cc @@ -22,6 +22,8 @@ limitations under the License. namespace tensorflow { namespace { +static GcsFileSystem::TimeoutConfig kTestTimeoutConfig(5, 1, 10, 20, 30); + class FakeAuthProvider : public AuthProvider { public: Status GetToken(string* token) override { @@ -35,12 +37,14 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) { {new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" "Auth Token: fake_token\n" - "Range: 0-5\n", + "Range: 0-5\n" + "Timeouts: 5 1 20\n", "012345"), new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" "Auth Token: fake_token\n" - "Range: 6-11\n", + "Range: 6-11\n" + "Timeouts: 5 1 20\n", "6789")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -49,7 +53,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -73,12 +77,14 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_differentN) { {new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" "Auth Token: fake_token\n" - "Range: 0-2\n", + "Range: 0-2\n" + "Timeouts: 5 1 20\n", "012"), new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" "Auth Token: fake_token\n" - "Range: 3-12\n", + "Range: 3-12\n" + "Timeouts: 5 1 20\n", "3456789")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -87,7 +93,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_differentN) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -116,26 +122,30 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache) { {new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" "Auth Token: fake_token\n" - "Range: 0-8\n", + "Range: 0-8\n" + "Timeouts: 5 1 20\n", "012345678"), new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" "Auth Token: fake_token\n" - "Range: 9-17\n", + "Range: 9-17\n" + "Timeouts: 5 1 20\n", "9abcde"), new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/random_access.txt\n" "Auth Token: fake_token\n" - "Range: 18-26\n", + "Range: 18-26\n" + "Timeouts: 5 1 20\n", "")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + 9 /* block size */, 18 /* max bytes */, + 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, + 0 /* initial retry delay */, kTestTimeoutConfig); char scratch[100]; StringPiece result; @@ -191,20 +201,23 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_MaxStaleness) { std::vector requests( {new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n" "Auth Token: fake_token\n" - "Range: 0-7\n", + "Range: 0-7\n" + "Timeouts: 5 1 20\n", "01234567"), new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n" "Auth Token: fake_token\n" - "Range: 8-15\n", + "Range: 8-15\n" + "Timeouts: 5 1 20\n", "89abcdef")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 8 /* block size */, 16 /* max bytes */, 3600 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + 8 /* block size */, 16 /* max bytes */, + 3600 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, + 0 /* initial retry delay */, kTestTimeoutConfig); char scratch[100]; StringPiece result; // There should only be two HTTP requests issued to GCS even though we iterate @@ -238,14 +251,15 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_MaxStaleness) { TEST(GcsFileSystemTest, NewRandomAccessFile_NoObjectName) { std::vector requests; - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* read ahead bytes */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + 0 /* read ahead bytes */, 0 /* max bytes */, + 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, + 0 /* initial retry delay */, kTestTimeoutConfig); std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -257,24 +271,28 @@ TEST(GcsFileSystemTest, NewWritableFile) { {new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n" "Auth Token: fake_token\n" - "Range: 0-7\n", + "Range: 0-7\n" + "Timeouts: 5 1 20\n", "01234567"), new FakeHttpRequest( "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" "uploadType=resumable&name=path%2Fwriteable\n" "Auth Token: fake_token\n" "Header X-Upload-Content-Length: 17\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "", {{"Location", "https://custom/upload/location"}}), new FakeHttpRequest("Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" "Header Content-Range: bytes 0-16/17\n" + "Timeouts: 5 1 30\n" "Put body: content1,content2\n", ""), new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n" "Auth Token: fake_token\n" - "Range: 0-7\n", + "Range: 0-7\n" + "Timeouts: 5 1 20\n", "01234567")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -283,7 +301,7 @@ TEST(GcsFileSystemTest, NewWritableFile) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); // Read from the file first, to fill the block cache. std::unique_ptr rfile; @@ -315,33 +333,39 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) { "uploadType=resumable&name=path%2Fwriteable.txt\n" "Auth Token: fake_token\n" "Header X-Upload-Content-Length: 17\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "", {{"Location", "https://custom/upload/location"}}), new FakeHttpRequest("Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" "Header Content-Range: bytes 0-16/17\n" + "Timeouts: 5 1 30\n" "Put body: content1,content2\n", "", errors::Unavailable("503"), 503), new FakeHttpRequest("Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Header Content-Range: bytes */17\n" "Put: yes\n", "", errors::FailedPrecondition("308"), nullptr, - {{"Range", "0-10"}}, 308), + {{"Range", "0-10"}}, 308, {}), new FakeHttpRequest("Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" "Header Content-Range: bytes 11-16/17\n" + "Timeouts: 5 1 30\n" "Put body: ntent2\n", "", errors::Unavailable("503"), 503), new FakeHttpRequest("Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Header Content-Range: bytes */17\n" "Put: yes\n", "", errors::FailedPrecondition("308"), nullptr, - {{"Range", "bytes=0-12"}}, 308), + {{"Range", "bytes=0-12"}}, 308, {}), new FakeHttpRequest("Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" "Header Content-Range: bytes 13-16/17\n" + "Timeouts: 5 1 30\n" "Put body: ent2\n", "")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -351,7 +375,7 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); std::unique_ptr file; TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); @@ -369,38 +393,44 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceedsOnGetStatus) { {new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n" "Auth Token: fake_token\n" - "Range: 0-7\n", + "Range: 0-7\n" + "Timeouts: 5 1 20\n", "01234567"), new FakeHttpRequest( "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" "uploadType=resumable&name=path%2Fwriteable\n" "Auth Token: fake_token\n" "Header X-Upload-Content-Length: 17\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "", {{"Location", "https://custom/upload/location"}}), new FakeHttpRequest("Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" "Header Content-Range: bytes 0-16/17\n" + "Timeouts: 5 1 30\n" "Put body: content1,content2\n", "", errors::Unavailable("503"), 503), new FakeHttpRequest("Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Header Content-Range: bytes */17\n" "Put: yes\n", - "", Status::OK(), nullptr, {}, 201), + "", Status::OK(), nullptr, {}, 201, {}), new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n" "Auth Token: fake_token\n" - "Range: 0-7\n", + "Range: 0-7\n" + "Timeouts: 5 1 20\n", "01234567")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 8 /* block size */, 8 /* max bytes */, 3600 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + 8 /* block size */, 8 /* max bytes */, + 3600 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, + 0 /* initial retry delay */, kTestTimeoutConfig); // Pull the file's first block into the cache. This will trigger the first // HTTP request to GCS. std::unique_ptr rfile; @@ -434,25 +464,29 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) { "uploadType=resumable&name=path%2Fwriteable.txt\n" "Auth Token: fake_token\n" "Header X-Upload-Content-Length: 17\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "", {{"Location", "https://custom/upload/location"}}), new FakeHttpRequest("Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" "Header Content-Range: bytes 0-16/17\n" + "Timeouts: 5 1 30\n" "Put body: content1,content2\n", "", errors::Unavailable("503"), 503)}); for (int i = 0; i < 10; i++) { requests.emplace_back(new FakeHttpRequest( "Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Header Content-Range: bytes */17\n" "Put: yes\n", "", errors::FailedPrecondition("important HTTP error 308"), nullptr, - {{"Range", "0-10"}}, 308)); + {{"Range", "0-10"}}, 308, {})); requests.emplace_back(new FakeHttpRequest( "Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" "Header Content-Range: bytes 11-16/17\n" + "Timeouts: 5 1 30\n" "Put body: ntent2\n", "", errors::Unavailable("important HTTP error 503"), 503)); } @@ -463,12 +497,14 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) { "uploadType=resumable&name=path%2Fwriteable.txt\n" "Auth Token: fake_token\n" "Header X-Upload-Content-Length: 17\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "", {{"Location", "https://custom/upload/location"}})); requests.emplace_back( new FakeHttpRequest("Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" "Header Content-Range: bytes 0-16/17\n" + "Timeouts: 5 1 30\n" "Put body: content1,content2\n", "")); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -478,7 +514,7 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 2 /* initial retry delay */); + 2 /* initial retry delay */, kTestTimeoutConfig); std::unique_ptr file; TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); @@ -500,11 +536,13 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) { "uploadType=resumable&name=path%2Fwriteable.txt\n" "Auth Token: fake_token\n" "Header X-Upload-Content-Length: 17\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "", {{"Location", "https://custom/upload/location"}}), new FakeHttpRequest("Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" "Header Content-Range: bytes 0-16/17\n" + "Timeouts: 5 1 30\n" "Put body: content1,content2\n", "", errors::NotFound("important HTTP error 410"), 410), @@ -515,11 +553,13 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) { "uploadType=resumable&name=path%2Fwriteable.txt\n" "Auth Token: fake_token\n" "Header X-Upload-Content-Length: 17\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "", {{"Location", "https://custom/upload/location"}}), new FakeHttpRequest("Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" "Header Content-Range: bytes 0-16/17\n" + "Timeouts: 5 1 30\n" "Put body: content1,content2\n", "")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -529,7 +569,7 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); std::unique_ptr file; TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); @@ -558,7 +598,7 @@ TEST(GcsFileSystemTest, NewWritableFile_NoObjectName) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -570,33 +610,38 @@ TEST(GcsFileSystemTest, NewAppendableFile) { {new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n" "Auth Token: fake_token\n" - "Range: 0-31\n", + "Range: 0-31\n" + "Timeouts: 5 1 20\n", "content1,"), new FakeHttpRequest( "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?" "uploadType=resumable&name=path%2Fappendable\n" "Auth Token: fake_token\n" "Header X-Upload-Content-Length: 17\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "", {{"Location", "https://custom/upload/location"}}), new FakeHttpRequest("Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" "Header Content-Range: bytes 0-16/17\n" + "Timeouts: 5 1 30\n" "Put body: content1,content2\n", ""), new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n" "Auth Token: fake_token\n" - "Range: 0-31\n", + "Range: 0-31\n" + "Timeouts: 5 1 20\n", "01234567")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 32 /* block size */, 32 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + 32 /* block size */, 32 /* max bytes */, + 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, + 0 /* initial retry delay */, kTestTimeoutConfig); // Create an appendable file. This should read the file from GCS, and pull its // contents into the block cache. @@ -629,7 +674,7 @@ TEST(GcsFileSystemTest, NewAppendableFile_NoObjectName) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -642,7 +687,8 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) { {new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Frandom_access.txt?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"", content.size(), "\", \"updated\": \"2016-04-29T23:15:24.896Z\"}")), new FakeHttpRequest( @@ -650,7 +696,7 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) { "path%2Frandom_access.txt\n" "Auth Token: fake_token\n" "Range: 0-", - content.size() - 1, "\n"), + content.size() - 1, "\n", "Timeouts: 5 1 20\n"), content)}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -659,7 +705,7 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); std::unique_ptr region; TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile( @@ -678,7 +724,7 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile_NoObjectName) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); std::unique_ptr region; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -689,7 +735,8 @@ TEST(GcsFileSystemTest, FileExists_YesAsObject) { std::vector requests({new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Ffile1.txt?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -699,7 +746,7 @@ TEST(GcsFileSystemTest, FileExists_YesAsObject) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt")); } @@ -709,13 +756,15 @@ TEST(GcsFileSystemTest, FileExists_YesAsFolder) { {new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Fsubfolder?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubfolder%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/subfolder/\" }]}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -725,7 +774,7 @@ TEST(GcsFileSystemTest, FileExists_YesAsFolder) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder")); } @@ -734,11 +783,13 @@ TEST(GcsFileSystemTest, FileExists_YesAsBucket) { std::vector requests( {new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"size\": \"100\"}"), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"size\": \"100\"}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -747,7 +798,7 @@ TEST(GcsFileSystemTest, FileExists_YesAsBucket) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); TF_EXPECT_OK(fs.FileExists("gs://bucket1")); TF_EXPECT_OK(fs.FileExists("gs://bucket1/")); @@ -758,13 +809,15 @@ TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) { {new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Ffile1.txt?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2Ffile1.txt%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": []}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -773,7 +826,7 @@ TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); EXPECT_EQ(errors::Code::NOT_FOUND, fs.FileExists("gs://bucket/path/file1.txt").code()); @@ -783,11 +836,13 @@ TEST(GcsFileSystemTest, FileExists_NotAsBucket) { std::vector requests( {new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket2\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket2\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -796,7 +851,7 @@ TEST(GcsFileSystemTest, FileExists_NotAsBucket) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, fs.FileExists("gs://bucket2/").code()); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -808,29 +863,33 @@ TEST(GcsFileSystemTest, FileExists_StatCache) { {new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Ffile1.txt?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}")), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Fsubfolder?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubfolder%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/subfolder/\" }]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 3600 /* stat cache max age */, + 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, + 0 /* initial retry delay */, kTestTimeoutConfig); // The stat cache will ensure that repeated lookups don't trigger additional // HTTP requests. @@ -845,7 +904,8 @@ TEST(GcsFileSystemTest, GetChildren_NoItems) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix=" "path%2F\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"prefixes\": [\"path/subpath/\"]}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -854,7 +914,7 @@ TEST(GcsFileSystemTest, GetChildren_NoItems) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -867,7 +927,8 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix=" "path%2F\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/file3.txt\" }]," @@ -879,7 +940,7 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -893,7 +954,8 @@ TEST(GcsFileSystemTest, GetChildren_SelfDirectoryMarker) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix=" "path%2F\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/\" }," " { \"name\": \"path/file3.txt\" }]," @@ -905,7 +967,7 @@ TEST(GcsFileSystemTest, GetChildren_SelfDirectoryMarker) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay */, kTestTimeoutConfig); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -918,7 +980,8 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles_NoSlash) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix=" "path%2F\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/file3.txt\" }]," @@ -930,7 +993,7 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles_NoSlash) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children)); @@ -943,7 +1006,8 @@ TEST(GcsFileSystemTest, GetChildren_Root) { std::vector requests({new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket-a-b-c/o?" "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -952,7 +1016,7 @@ TEST(GcsFileSystemTest, GetChildren_Root) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket-a-b-c", &children)); @@ -965,7 +1029,8 @@ TEST(GcsFileSystemTest, GetChildren_Empty) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&prefix=" "path%2F\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -974,7 +1039,7 @@ TEST(GcsFileSystemTest, GetChildren_Empty) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -988,7 +1053,8 @@ TEST(GcsFileSystemTest, GetChildren_Pagination) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&" "prefix=path%2F\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"nextPageToken\": \"ABCD==\", " "\"items\": [ " " { \"name\": \"path/file1.txt\" }," @@ -999,7 +1065,8 @@ TEST(GcsFileSystemTest, GetChildren_Pagination) { "fields=items%2Fname%2Cprefixes%2CnextPageToken&delimiter=%2F&" "prefix=path%2F" "&pageToken=ABCD==\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/file4.txt\" }," " { \"name\": \"path/file5.txt\" }]}")}); @@ -1011,7 +1078,7 @@ TEST(GcsFileSystemTest, GetChildren_Pagination) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children)); @@ -1025,7 +1092,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_NoWildcard) { std::vector requests({new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/subpath/file2.txt\" }]}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -1035,7 +1103,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_NoWildcard) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); std::vector result; TF_EXPECT_OK( @@ -1048,7 +1116,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_BucketAndWildcard) { std::vector requests({new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," @@ -1060,7 +1129,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_BucketAndWildcard) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", &result)); @@ -1074,7 +1143,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_Matches) { std::vector requests({new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," @@ -1086,7 +1156,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_Matches) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file2.txt", &result)); @@ -1098,7 +1168,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_SelfDirectoryMarker) { std::vector requests({new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/\" }," " { \"name\": \"path/file3.txt\" }]}")}); @@ -1109,7 +1180,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_SelfDirectoryMarker) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", &result)); @@ -1120,7 +1191,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) { std::vector requests({new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," @@ -1132,7 +1204,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file3.txt", &result)); @@ -1148,7 +1220,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_OnlyWildcard) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); std::vector result; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -1160,13 +1232,15 @@ TEST(GcsFileSystemTest, GetMatchingPaths_Cache) { {new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/subpath/file2.txt\" }]}"), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," @@ -1178,7 +1252,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_Cache) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 3600 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); // Repeated calls to fs.GetMatchingPaths on these patterns should not lead to // any additional HTTP requests to GCS. @@ -1201,26 +1275,30 @@ TEST(GcsFileSystemTest, DeleteFile) { {new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n" "Auth Token: fake_token\n" - "Range: 0-15\n", + "Range: 0-15\n" + "Timeouts: 5 1 20\n", "01234567"), new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" "/bucket/o/path%2Ffile1.txt\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", ""), new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n" "Auth Token: fake_token\n" - "Range: 0-15\n", + "Range: 0-15\n" + "Timeouts: 5 1 20\n", "76543210")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 16 /* block size */, 16 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + 16 /* block size */, 16 /* max bytes */, + 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, + 0 /* initial retry delay*/, kTestTimeoutConfig); // Do an initial read of the file to load its contents into the block cache. char scratch[100]; @@ -1246,7 +1324,7 @@ TEST(GcsFileSystemTest, DeleteFile_NoObjectName) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, fs.DeleteFile("gs://bucket/").code()); @@ -1256,7 +1334,8 @@ TEST(GcsFileSystemTest, DeleteDir_Empty) { std::vector requests({new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -1265,7 +1344,7 @@ TEST(GcsFileSystemTest, DeleteDir_Empty) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/")); } @@ -1275,12 +1354,14 @@ TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) { {new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/\" }]}"), new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" "/bucket/o/path%2F\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", "")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -1290,7 +1371,7 @@ TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/")); } @@ -1298,7 +1379,8 @@ TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) { TEST(GcsFileSystemTest, DeleteDir_BucketOnly) { std::vector requests({new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?fields=items%2F" - "name%2CnextPageToken&maxResults=2\nAuth Token: fake_token\n", + "name%2CnextPageToken&maxResults=2\nAuth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -1307,7 +1389,7 @@ TEST(GcsFileSystemTest, DeleteDir_BucketOnly) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); TF_EXPECT_OK(fs.DeleteDir("gs://bucket")); } @@ -1316,7 +1398,8 @@ TEST(GcsFileSystemTest, DeleteDir_NonEmpty) { std::vector requests({new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2F&maxResults=2\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/file1.txt\" }]}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -1326,7 +1409,7 @@ TEST(GcsFileSystemTest, DeleteDir_NonEmpty) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); EXPECT_EQ(error::Code::FAILED_PRECONDITION, fs.DeleteDir("gs://bucket/path/").code()); @@ -1336,7 +1419,8 @@ TEST(GcsFileSystemTest, GetFileSize) { std::vector requests({new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "file.txt?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -1346,7 +1430,7 @@ TEST(GcsFileSystemTest, GetFileSize) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); uint64 size; TF_EXPECT_OK(fs.GetFileSize("gs://bucket/file.txt", &size)); @@ -1362,7 +1446,7 @@ TEST(GcsFileSystemTest, GetFileSize_NoObjectName) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); uint64 size; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -1376,14 +1460,16 @@ TEST(GcsFileSystemTest, RenameFile_Folder) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path1%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path1/subfolder/file1.txt\" }]}"), // Requesting the full list of files in the folder. new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path1%2F\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path1/\" }," // A directory marker. " { \"name\": \"path1/subfolder/file1.txt\" }," @@ -1393,13 +1479,15 @@ TEST(GcsFileSystemTest, RenameFile_Folder) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path1%2F/rewriteTo/b/bucket/o/path2%2F\n" "Auth Token: fake_token\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "{\"done\": true}"), // Deleting the original directory marker. new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path1%2F\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", ""), // Copying the first file. @@ -1408,13 +1496,15 @@ TEST(GcsFileSystemTest, RenameFile_Folder) { "path1%2Fsubfolder%2Ffile1.txt/rewriteTo/b/bucket/o/" "path2%2Fsubfolder%2Ffile1.txt\n" "Auth Token: fake_token\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "{\"done\": true}"), // Deleting the first original file. new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path1%2Fsubfolder%2Ffile1.txt\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", ""), // Copying the second file. @@ -1422,13 +1512,15 @@ TEST(GcsFileSystemTest, RenameFile_Folder) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path1%2Ffile2.txt/rewriteTo/b/bucket/o/path2%2Ffile2.txt\n" "Auth Token: fake_token\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "{\"done\": true}"), // Deleting the second original file. new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path1%2Ffile2.txt\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", "")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -1438,7 +1530,7 @@ TEST(GcsFileSystemTest, RenameFile_Folder) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); TF_EXPECT_OK(fs.RenameFile("gs://bucket/path1", "gs://bucket/path2/")); } @@ -1448,25 +1540,29 @@ TEST(GcsFileSystemTest, RenameFile_Object) { {new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n" "Auth Token: fake_token\n" - "Range: 0-15\n", + "Range: 0-15\n" + "Timeouts: 5 1 20\n", "01234567"), new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n" "Auth Token: fake_token\n" - "Range: 0-15\n", + "Range: 0-15\n" + "Timeouts: 5 1 20\n", "76543210"), // IsDirectory is checking whether there are children objects. new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{}"), // IsDirectory is checking if the path exists as an object. new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Fsrc.txt?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}")), // Copying to the new location. @@ -1474,33 +1570,38 @@ TEST(GcsFileSystemTest, RenameFile_Object) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n" "Auth Token: fake_token\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "{\"done\": true}"), // Deleting the original file. new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Fsrc.txt\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", ""), new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n" "Auth Token: fake_token\n" - "Range: 0-15\n", + "Range: 0-15\n" + "Timeouts: 5 1 20\n", "89abcdef"), new FakeHttpRequest( "Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n" "Auth Token: fake_token\n" - "Range: 0-15\n", + "Range: 0-15\n" + "Timeouts: 5 1 20\n", "fedcba98")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 16 /* block size */, 64 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + 16 /* block size */, 64 /* max bytes */, + 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, + 0 /* initial retry delay*/, kTestTimeoutConfig); // Do an initial read of the source and destination files to load their // contents into the block cache. char scratch[100]; @@ -1531,13 +1632,15 @@ TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{}"), // IsDirectory is checking if the path exists as an object. new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Fsrc.txt?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}")), // Copying to the new location. @@ -1545,13 +1648,15 @@ TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n" "Auth Token: fake_token\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "{\"done\": true}"), // Deleting the original file - the deletion returns a failure. new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Fsrc.txt\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", "", errors::Unavailable("503"), 503), // Deleting the original file again - the deletion returns NOT_FOUND. @@ -1559,6 +1664,7 @@ TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Fsrc.txt\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", "", errors::NotFound("404"), 404)}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -1568,7 +1674,7 @@ TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); TF_EXPECT_OK( fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt")); @@ -1582,13 +1688,15 @@ TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{}"), // IsDirectory is checking if the path exists as an object. new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Fsrc.txt?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}")), // Copying to the new location. @@ -1596,7 +1704,8 @@ TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n" "Auth Token: fake_token\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "{\"done\": false}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -1605,7 +1714,7 @@ TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); EXPECT_EQ( errors::Code::UNIMPLEMENTED, @@ -1617,7 +1726,8 @@ TEST(GcsFileSystemTest, Stat_Object) { std::vector requests({new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "file.txt?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -1627,7 +1737,7 @@ TEST(GcsFileSystemTest, Stat_Object) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat)); @@ -1641,13 +1751,15 @@ TEST(GcsFileSystemTest, Stat_Folder) { {new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "subfolder?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"subfolder/\" }]}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -1657,7 +1769,7 @@ TEST(GcsFileSystemTest, Stat_Folder) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", &stat)); @@ -1671,13 +1783,15 @@ TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) { {new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -1686,7 +1800,7 @@ TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); FileStatistics stat; EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/path", &stat).code()); @@ -1695,7 +1809,8 @@ TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) { TEST(GcsFileSystemTest, Stat_Bucket) { std::vector requests({new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -1704,7 +1819,7 @@ TEST(GcsFileSystemTest, Stat_Bucket) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/", &stat)); @@ -1716,7 +1831,8 @@ TEST(GcsFileSystemTest, Stat_Bucket) { TEST(GcsFileSystemTest, Stat_BucketNotFound) { std::vector requests({new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -1725,7 +1841,7 @@ TEST(GcsFileSystemTest, Stat_BucketNotFound) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); FileStatistics stat; EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/", &stat).code()); @@ -1736,29 +1852,33 @@ TEST(GcsFileSystemTest, Stat_Cache) { {new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "file.txt?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}")), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "subfolder?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"subfolder/\" }]}")}); - GcsFileSystem fs( - std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, 0 /* initial retry delay */); + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 3600 /* stat cache max age */, + 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, + 0 /* initial retry delay*/, kTestTimeoutConfig); // Repeated calls to fs.Stat on these paths should not lead to any additional // HTTP requests to GCS. @@ -1781,12 +1901,14 @@ TEST(GcsFileSystemTest, IsDirectory_NotFound) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=file.txt%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{}"), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "file.txt?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -1795,7 +1917,7 @@ TEST(GcsFileSystemTest, IsDirectory_NotFound) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/file.txt").code()); @@ -1807,12 +1929,14 @@ TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=file.txt%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{}"), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "file.txt?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -1822,7 +1946,7 @@ TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); EXPECT_EQ(error::Code::FAILED_PRECONDITION, fs.IsDirectory("gs://bucket/file.txt").code()); @@ -1834,13 +1958,15 @@ TEST(GcsFileSystemTest, IsDirectory_Yes) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [{\"name\": \"subfolder/\"}]}"), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=subfolder%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [{\"name\": \"subfolder/\"}]}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -1849,7 +1975,7 @@ TEST(GcsFileSystemTest, IsDirectory_Yes) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder")); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder/")); @@ -1859,11 +1985,13 @@ TEST(GcsFileSystemTest, IsDirectory_Bucket) { std::vector requests( {new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{}"), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{}")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -1872,7 +2000,7 @@ TEST(GcsFileSystemTest, IsDirectory_Bucket) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); TF_EXPECT_OK(fs.IsDirectory("gs://bucket")); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/")); @@ -1881,7 +2009,8 @@ TEST(GcsFileSystemTest, IsDirectory_Bucket) { TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) { std::vector requests({new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -1890,7 +2019,7 @@ TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/").code()); } @@ -1902,10 +2031,12 @@ TEST(GcsFileSystemTest, CreateDir_Folder) { "uploadType=resumable&name=subpath%2F\n" "Auth Token: fake_token\n" "Header X-Upload-Content-Length: 0\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "", {{"Location", "https://custom/upload/location"}}), new FakeHttpRequest("Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 30\n" "Put body: \n", ""), new FakeHttpRequest( @@ -1913,10 +2044,12 @@ TEST(GcsFileSystemTest, CreateDir_Folder) { "uploadType=resumable&name=subpath%2F\n" "Auth Token: fake_token\n" "Header X-Upload-Content-Length: 0\n" - "Post: yes\n", + "Post: yes\n" + "Timeouts: 5 1 10\n", "", {{"Location", "https://custom/upload/location"}}), new FakeHttpRequest("Uri: https://custom/upload/location\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 30\n" "Put body: \n", "")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -1926,7 +2059,7 @@ TEST(GcsFileSystemTest, CreateDir_Folder) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath")); TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath/")); @@ -1936,11 +2069,13 @@ TEST(GcsFileSystemTest, CreateDir_Bucket) { std::vector requests( {new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", ""), new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -1949,7 +2084,7 @@ TEST(GcsFileSystemTest, CreateDir_Bucket) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); TF_EXPECT_OK(fs.CreateDir("gs://bucket/")); TF_EXPECT_OK(fs.CreateDir("gs://bucket")); @@ -1962,14 +2097,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_Ok) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/file1.txt\" }]}"), // GetChildren recursively. new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/\" }," // The current directory's marker. " { \"name\": \"path/file1.txt\" }," @@ -1979,30 +2116,35 @@ TEST(GcsFileSystemTest, DeleteRecursively_Ok) { new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" "/bucket/o/path%2F\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", ""), // Delete the object - fails and will be retried. new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" "/bucket/o/path%2Ffile1.txt\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", "", errors::Unavailable("500"), 500), // Delete the object again. new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" "/bucket/o/path%2Ffile1.txt\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", ""), // Delete the object. new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" "/bucket/o/path%2Fsubpath%2Ffile2.txt\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", ""), // Delete the object. new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" "/bucket/o/path%2Ffile3.txt\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", "")}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -2012,7 +2154,7 @@ TEST(GcsFileSystemTest, DeleteRecursively_Ok) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); int64 undeleted_files, undeleted_dirs; TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files, @@ -2028,14 +2170,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/file1.txt\" }]}"), // Calling GetChildren recursively. new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2F\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/\" }," @@ -2045,12 +2189,14 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) { new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" "/bucket/o/path%2Ffile1.txt\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", ""), // Deleting the directory marker gs://bucket/path/ - fails with 404. new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" "/bucket/o/path%2Fsubpath%2F\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", "", errors::NotFound("404"), 404), // Checking if gs://bucket/path/subpath/ is a folder - it is. @@ -2058,19 +2204,22 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsubpath%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", strings::StrCat("{\"items\": [ " " { \"name\": \"path/subpath/\" }]}")), // Deleting the object gs://bucket/path/subpath/file2.txt new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" "/bucket/o/path%2Fsubpath%2Ffile2.txt\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", ""), // Deleting the object s://bucket/path/file3.txt - fails with 404. new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b" "/bucket/o/path%2Ffile3.txt\n" "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" "Delete: yes\n", "", errors::NotFound("404"), 404), // Checking if gs://bucket/path/file3.txt/ is a folder - it's not. @@ -2078,13 +2227,15 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2Ffile3.txt%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{}"), // Checking if gs://bucket/path/file3.txt is an object - fails with 404. new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path%2Ffile3.txt?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), @@ -2094,7 +2245,7 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); int64 undeleted_files, undeleted_dirs; TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files, @@ -2110,13 +2261,15 @@ TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) { "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" "fields=items%2Fname%2CnextPageToken&prefix=path%2F" "&maxResults=1\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "{}"), // IsDirectory is checking if the path exists as an object. new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" "path?fields=size%2Cupdated\n" - "Auth Token: fake_token\n", + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), std::unique_ptr( @@ -2125,7 +2278,7 @@ TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) { 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, - 0 /* initial retry delay */); + 0 /* initial retry delay*/, kTestTimeoutConfig); int64 undeleted_files, undeleted_dirs; EXPECT_EQ(error::Code::NOT_FOUND, @@ -2142,6 +2295,11 @@ TEST(GcsFileSystemTest, OverrideCacheParameters) { EXPECT_EQ(128 * 1024 * 1024, fs1.block_size()); EXPECT_EQ(2 * fs1.block_size(), fs1.max_bytes()); EXPECT_EQ(0, fs1.max_staleness()); + EXPECT_EQ(120, fs1.timeouts().connect); + EXPECT_EQ(60, fs1.timeouts().idle); + EXPECT_EQ(3600, fs1.timeouts().metadata); + EXPECT_EQ(3600, fs1.timeouts().read); + EXPECT_EQ(3600, fs1.timeouts().write); // Verify legacy readahead buffer override sets block size. setenv("GCS_READAHEAD_BUFFER_SIZE_BYTES", "123456789", 1); @@ -2167,6 +2325,19 @@ TEST(GcsFileSystemTest, OverrideCacheParameters) { EXPECT_EQ(32, fs4.stat_cache_max_entries()); EXPECT_EQ(30, fs4.matching_paths_cache_max_age()); EXPECT_EQ(64, fs4.matching_paths_cache_max_entries()); + + // Verify timeout overrides. + setenv("GCS_REQUEST_CONNECTION_TIMEOUT_SECS", "10", 1); + setenv("GCS_REQUEST_IDLE_TIMEOUT_SECS", "5", 1); + setenv("GCS_METADATA_REQUEST_TIMEOUT_SECS", "20", 1); + setenv("GCS_READ_REQUEST_TIMEOUT_SECS", "30", 1); + setenv("GCS_WRITE_REQUEST_TIMEOUT_SECS", "40", 1); + GcsFileSystem fs5; + EXPECT_EQ(10, fs5.timeouts().connect); + EXPECT_EQ(5, fs5.timeouts().idle); + EXPECT_EQ(20, fs5.timeouts().metadata); + EXPECT_EQ(30, fs5.timeouts().read); + EXPECT_EQ(40, fs5.timeouts().write); } } // namespace diff --git a/tensorflow/core/platform/cloud/http_request.h b/tensorflow/core/platform/cloud/http_request.h index 02d9e9054a..95a436c622 100644 --- a/tensorflow/core/platform/cloud/http_request.h +++ b/tensorflow/core/platform/cloud/http_request.h @@ -118,6 +118,16 @@ class HttpRequest { // Url encodes str and returns a new string. virtual string EscapeString(const string& str) = 0; + /// \brief Set timeouts for this request. + /// + /// The connection parameter controls how long we should wait for the + /// connection to be established. The inactivity parameter controls how long + /// we should wait between additional responses from the server. Finally the + /// total parameter controls the maximum total connection time to prevent + /// hanging indefinitely. + virtual Status SetTimeouts(uint32 connection, uint32 inactivity, + uint32 total) = 0; + TF_DISALLOW_COPY_AND_ASSIGN(HttpRequest); }; diff --git a/tensorflow/core/platform/cloud/http_request_fake.h b/tensorflow/core/platform/cloud/http_request_fake.h index bfe04f6363..22398d739c 100644 --- a/tensorflow/core/platform/cloud/http_request_fake.h +++ b/tensorflow/core/platform/cloud/http_request_fake.h @@ -37,13 +37,14 @@ class FakeHttpRequest : public CurlHttpRequest { public: /// Return the response for the given request. FakeHttpRequest(const string& request, const string& response) - : FakeHttpRequest(request, response, Status::OK(), nullptr, {}, 200) {} + : FakeHttpRequest(request, response, Status::OK(), nullptr, {}, 200, {}) { + } /// Return the response with headers for the given request. FakeHttpRequest(const string& request, const string& response, const std::map& response_headers) : FakeHttpRequest(request, response, Status::OK(), nullptr, - response_headers, 200) {} + response_headers, 200, {}) {} /// \brief Return the response for the request and capture the POST body. /// @@ -51,13 +52,13 @@ class FakeHttpRequest : public CurlHttpRequest { FakeHttpRequest(const string& request, const string& response, string* captured_post_body) : FakeHttpRequest(request, response, Status::OK(), captured_post_body, {}, - 200) {} + 200, {}) {} /// \brief Return the response and the status for the given request. FakeHttpRequest(const string& request, const string& response, Status response_status, uint64 response_code) : FakeHttpRequest(request, response, response_status, nullptr, {}, - response_code) {} + response_code, {}) {} /// \brief Return the response and the status for the given request /// and capture the POST body. @@ -66,13 +67,15 @@ class FakeHttpRequest : public CurlHttpRequest { FakeHttpRequest(const string& request, const string& response, Status response_status, string* captured_post_body, const std::map& response_headers, - uint64 response_code) + uint64 response_code, + absl::optional> timeouts) : expected_request_(request), response_(response), response_status_(response_status), captured_post_body_(captured_post_body), response_headers_(response_headers), - response_code_(response_code) {} + response_code_(response_code), + timeouts_(timeouts) {} Status Init() override { return Status::OK(); } Status SetUri(const string& uri) override { @@ -160,6 +163,13 @@ class FakeHttpRequest : public CurlHttpRequest { virtual uint64 GetResponseCode() const override { return response_code_; } + Status SetTimeouts(uint32 connection, uint32 inactivity, + uint32 total) override { + actual_request_ += strings::StrCat("Timeouts: ", connection, " ", + inactivity, " ", total, "\n"); + return Status::OK(); + } + private: std::vector* buffer_ = nullptr; string expected_request_; @@ -169,6 +179,7 @@ class FakeHttpRequest : public CurlHttpRequest { string* captured_post_body_ = nullptr; std::map response_headers_; uint64 response_code_ = 0; + absl::optional> timeouts_; }; /// Fake HttpRequest factory for testing. -- cgit v1.2.3