From 8ec17cff881923fe3ff26b7de812cf0b2d21cd07 Mon Sep 17 00:00:00 2001 From: Michael Banfield Date: Thu, 2 Aug 2018 13:13:48 -0700 Subject: Add a optional bucket location check to the GCS Filesystem. PiperOrigin-RevId: 207152562 --- tensorflow/core/platform/cloud/BUILD | 1 + tensorflow/core/platform/cloud/gcs_file_system.cc | 110 +- tensorflow/core/platform/cloud/gcs_file_system.h | 37 +- .../core/platform/cloud/gcs_file_system_test.cc | 1425 ++++++++++++-------- 4 files changed, 965 insertions(+), 608 deletions(-) diff --git a/tensorflow/core/platform/cloud/BUILD b/tensorflow/core/platform/cloud/BUILD index 549996aaf8..647a797b82 100644 --- a/tensorflow/core/platform/cloud/BUILD +++ b/tensorflow/core/platform/cloud/BUILD @@ -74,6 +74,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":compute_engine_metadata_client", + ":compute_engine_zone_provider", ":curl_http_request", ":expiring_lru_cache", ":file_block_cache", diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc index 2e8d13acd5..67c872ac67 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system.cc @@ -57,6 +57,7 @@ constexpr char kGcsUriBase[] = "https://www.googleapis.com/storage/v1/"; constexpr char kGcsUploadUriBase[] = "https://www.googleapis.com/upload/storage/v1/"; constexpr char kStorageHost[] = "storage.googleapis.com"; +constexpr char kBucketMetadataLocationKey[] = "location"; constexpr size_t kReadAppendableFileBufferSize = 1024 * 1024; // In bytes. constexpr int kGetChildrenDefaultPageSize = 1000; // The HTTP response code "308 Resume Incomplete". @@ -98,6 +99,11 @@ constexpr uint64 kMatchingPathsCacheDefaultMaxAge = 0; constexpr char kMatchingPathsCacheMaxEntries[] = "GCS_MATCHING_PATHS_CACHE_MAX_ENTRIES"; constexpr size_t kMatchingPathsCacheDefaultMaxEntries = 1024; +// Number of bucket locations cached, most workloads wont touch more than one +// bucket so this limit is set fairly low +constexpr size_t kBucketLocationCacheMaxEntries = 10; +// ExpiringLRUCache doesnt support any "cache forever" option +constexpr size_t kCacheNeverExpire = std::numeric_limits::max(); // The file statistics returned by Stat() for directories. const FileStatistics DIRECTORY_STAT(0, 0, true); // Some environments exhibit unreliable DNS resolution. Set this environment @@ -131,6 +137,14 @@ constexpr char kTokensPerRequest[] = "GCS_TOKENS_PER_REQUEST"; // The environment variable to configure the initial tokens (format: ) constexpr char kInitialTokens[] = "GCS_INITIAL_TOKENS"; +// The environment variable to customize which GCS bucket locations are allowed, +// if the list is empty defaults to using the region of the zone (format, comma +// delimited list). Requires 'storage.buckets.get' permission. +constexpr char kAllowedBucketLocations[] = "GCS_ALLOWED_BUCKET_LOCATIONS"; +// When this value is passed as an allowed location detects the zone tensorflow +// is running in and restricts to buckets in that region. +constexpr char kDetectZoneSentinalValue[] = "auto"; + // TODO: DO NOT use a hardcoded path Status GetTmpFilename(string* filename) { #ifndef _WIN32 @@ -603,6 +617,19 @@ bool StringPieceIdentity(StringPiece str, StringPiece* value) { return true; } +/// \brief Utility function to split a comma delimited list of strings to an +/// unordered set +bool SplitByCommaToSet(StringPiece list, std::unordered_set* set) { + std::vector vector = str_util::Split(list, ","); + *set = std::unordered_set(vector.begin(), vector.end()); + return true; +} + +// \brief Convert Compute Engine zone to region +string ZoneToRegion(string* zone) { + return zone->substr(0, zone->find_last_of('-')); +} + } // namespace GcsFileSystem::GcsFileSystem() { @@ -616,6 +643,8 @@ GcsFileSystem::GcsFileSystem() { std::make_shared(http_request_factory_); auth_provider_ = std::unique_ptr( new GoogleAuthProvider(compute_engine_metadata_client_)); + zone_provider_ = std::unique_ptr( + new ComputeEngineZoneProvider(compute_engine_metadata_client_)); // Apply the sys env override for the readahead buffer size if it's provided. if (GetEnvVar(kReadaheadBufferSize, strings::safe_strtou64, &value)) { @@ -666,6 +695,9 @@ GcsFileSystem::GcsFileSystem() { matching_paths_cache_.reset(new ExpiringLRUCache>( matching_paths_cache_max_age, matching_paths_cache_max_entries)); + bucket_location_cache_.reset(new ExpiringLRUCache( + kCacheNeverExpire, kBucketLocationCacheMaxEntries)); + int64 resolve_frequency_secs; if (GetEnvVar(kResolveCacheSecs, strings::safe_strto64, &resolve_frequency_secs)) { @@ -745,24 +777,30 @@ GcsFileSystem::GcsFileSystem() { } throttle_.SetConfig(config); } + + GetEnvVar(kAllowedBucketLocations, SplitByCommaToSet, &allowed_locations_); } GcsFileSystem::GcsFileSystem( std::unique_ptr auth_provider, std::unique_ptr http_request_factory, - size_t block_size, size_t max_bytes, uint64 max_staleness, - uint64 stat_cache_max_age, size_t stat_cache_max_entries, - uint64 matching_paths_cache_max_age, + std::unique_ptr zone_provider, size_t block_size, + size_t max_bytes, uint64 max_staleness, uint64 stat_cache_max_age, + size_t stat_cache_max_entries, uint64 matching_paths_cache_max_age, size_t matching_paths_cache_max_entries, int64 initial_retry_delay_usec, - TimeoutConfig timeouts, + TimeoutConfig timeouts, const std::unordered_set& allowed_locations, std::pair* additional_header) : auth_provider_(std::move(auth_provider)), http_request_factory_(std::move(http_request_factory)), + zone_provider_(std::move(zone_provider)), file_block_cache_( MakeFileBlockCache(block_size, max_bytes, max_staleness)), stat_cache_(new StatCache(stat_cache_max_age, stat_cache_max_entries)), matching_paths_cache_(new MatchingPathsCache( matching_paths_cache_max_age, matching_paths_cache_max_entries)), + bucket_location_cache_(new BucketLocationCache( + kCacheNeverExpire, kBucketLocationCacheMaxEntries)), + allowed_locations_(allowed_locations), timeouts_(timeouts), initial_retry_delay_usec_(initial_retry_delay_usec), additional_header_(additional_header) {} @@ -771,6 +809,7 @@ Status GcsFileSystem::NewRandomAccessFile( const string& fname, std::unique_ptr* result) { string bucket, object; TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object)); + TF_RETURN_IF_ERROR(CheckBucketLocationConstraint(bucket)); result->reset(new GcsRandomAccessFile(fname, [this, bucket, object]( const string& fname, uint64 offset, size_t n, @@ -1072,11 +1111,7 @@ Status GcsFileSystem::StatForObject(const string& fname, const string& bucket, } Status GcsFileSystem::BucketExists(const string& bucket, bool* result) { - std::unique_ptr request; - TF_RETURN_IF_ERROR(CreateHttpRequest(&request)); - request->SetUri(strings::StrCat(kGcsUriBase, "b/", bucket)); - request->SetTimeouts(timeouts_.connect, timeouts_.idle, timeouts_.metadata); - const Status status = request->Send(); + const Status status = GetBucketMetadata(bucket, nullptr); switch (status.code()) { case errors::Code::OK: *result = true; @@ -1089,6 +1124,62 @@ Status GcsFileSystem::BucketExists(const string& bucket, bool* result) { } } +Status GcsFileSystem::CheckBucketLocationConstraint(const string& bucket) { + if (allowed_locations_.empty()) { + return Status::OK(); + } + + // Avoid calling external API's in the constructor + if (allowed_locations_.erase(kDetectZoneSentinalValue) == 1) { + string zone; + TF_RETURN_IF_ERROR(zone_provider_->GetZone(&zone)); + allowed_locations_.insert(ZoneToRegion(&zone)); + } + + string location; + TF_RETURN_IF_ERROR(GetBucketLocation(bucket, &location)); + if (allowed_locations_.find(location) != allowed_locations_.end()) { + return Status::OK(); + } + + return errors::FailedPrecondition(strings::Printf( + "Bucket '%s' is in '%s' location, allowed locations are: (%s).", + bucket.c_str(), location.c_str(), + str_util::Join(allowed_locations_, ", ").c_str())); +} + +Status GcsFileSystem::GetBucketLocation(const string& bucket, + string* location) { + auto compute_func = [this](const string& bucket, string* location) { + std::vector result_buffer; + Status status = GetBucketMetadata(bucket, &result_buffer); + Json::Value result; + TF_RETURN_IF_ERROR(ParseJson(result_buffer, &result)); + TF_RETURN_IF_ERROR( + GetStringValue(result, kBucketMetadataLocationKey, location)); + return Status::OK(); + }; + + TF_RETURN_IF_ERROR( + bucket_location_cache_->LookupOrCompute(bucket, location, compute_func)); + + return Status::OK(); +} + +Status GcsFileSystem::GetBucketMetadata(const string& bucket, + std::vector* result_buffer) { + std::unique_ptr request; + TF_RETURN_IF_ERROR(CreateHttpRequest(&request)); + request->SetUri(strings::StrCat(kGcsUriBase, "b/", bucket)); + + if (result_buffer != nullptr) { + request->SetResultBuffer(result_buffer); + } + + request->SetTimeouts(timeouts_.connect, timeouts_.idle, timeouts_.metadata); + return request->Send(); +} + Status GcsFileSystem::FolderExists(const string& dirname, bool* result) { StatCache::ComputeFunc compute_func = [this](const string& dirname, GcsFileStat* stat) { @@ -1514,6 +1605,7 @@ void GcsFileSystem::FlushCaches() { file_block_cache_->Flush(); stat_cache_->Clear(); matching_paths_cache_->Clear(); + bucket_location_cache_->Clear(); } void GcsFileSystem::SetStats(GcsStatsInterface* stats) { diff --git a/tensorflow/core/platform/cloud/gcs_file_system.h b/tensorflow/core/platform/cloud/gcs_file_system.h index a0372286f5..71db707687 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.h +++ b/tensorflow/core/platform/cloud/gcs_file_system.h @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/cloud/auth_provider.h" #include "tensorflow/core/platform/cloud/compute_engine_metadata_client.h" +#include "tensorflow/core/platform/cloud/compute_engine_zone_provider.h" #include "tensorflow/core/platform/cloud/expiring_lru_cache.h" #include "tensorflow/core/platform/cloud/file_block_cache.h" #include "tensorflow/core/platform/cloud/gcs_dns_cache.h" @@ -81,14 +82,19 @@ class GcsFileSystem : public FileSystem { public: struct TimeoutConfig; + // Main constructor used (via RetryingFileSystem) throughout Tensorflow GcsFileSystem(); + // Used mostly for unit testing or use cases which need to customize the + // filesystem from defaults GcsFileSystem(std::unique_ptr auth_provider, std::unique_ptr http_request_factory, - size_t block_size, size_t max_bytes, uint64 max_staleness, + std::unique_ptr zone_provider, size_t block_size, + size_t max_bytes, uint64 max_staleness, uint64 stat_cache_max_age, size_t stat_cache_max_entries, uint64 matching_paths_cache_max_age, size_t matching_paths_cache_max_entries, int64 initial_retry_delay_usec, TimeoutConfig timeouts, + const std::unordered_set& allowed_locations, std::pair* additional_header); Status NewRandomAccessFile( @@ -149,6 +155,9 @@ class GcsFileSystem : public FileSystem { return file_block_cache_->max_staleness(); } TimeoutConfig timeouts() const { return timeouts_; } + std::unordered_set allowed_locations() const { + return allowed_locations_; + } string additional_header_name() const { return additional_header_ ? additional_header_->first : ""; } @@ -230,6 +239,27 @@ class GcsFileSystem : public FileSystem { /// 'result' is set if the function returns OK. 'result' cannot be nullptr. Status BucketExists(const string& bucket, bool* result); + /// \brief Retrieves the GCS bucket location. Returns OK if the location was + /// retrieved. + /// + /// Given a string bucket the GCS bucket metadata API will be called and the + /// location string filled with the location of the bucket. + /// + /// This requires the bucket metadata permission. + /// Repeated calls for the same bucket are cached so this function can be + /// called frequently without causing an extra API call + Status GetBucketLocation(const string& bucket, string* location); + + /// \brief Check if the GCS buckets location is allowed with the current + /// constraint configuration + Status CheckBucketLocationConstraint(const string& bucket); + + /// \brief Given the input bucket `bucket`, fills `result_buffer` with the + /// results of the metadata. Returns OK if the API call succeeds without + /// error. + Status GetBucketMetadata(const string& bucket, + std::vector* result_buffer); + /// \brief Checks if the object exists. Returns OK if the check succeeded. /// /// 'result' is set if the function returns OK. 'result' cannot be nullptr. @@ -277,6 +307,7 @@ class GcsFileSystem : public FileSystem { mutex mu_; std::unique_ptr auth_provider_ GUARDED_BY(mu_); std::shared_ptr http_request_factory_; + std::unique_ptr zone_provider_; // block_cache_lock_ protects the file_block_cache_ pointer (Note that // FileBlockCache instances are themselves threadsafe). mutex block_cache_lock_; @@ -292,6 +323,10 @@ class GcsFileSystem : public FileSystem { using MatchingPathsCache = ExpiringLRUCache>; std::unique_ptr matching_paths_cache_; + using BucketLocationCache = ExpiringLRUCache; + std::unique_ptr bucket_location_cache_; + std::unordered_set allowed_locations_; + TimeoutConfig timeouts_; GcsStatsInterface* stats_ = nullptr; // Not owned. diff --git a/tensorflow/core/platform/cloud/gcs_file_system_test.cc b/tensorflow/core/platform/cloud/gcs_file_system_test.cc index e791ae5a19..ee2b034d74 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system_test.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system_test.cc @@ -24,6 +24,13 @@ namespace tensorflow { namespace { static GcsFileSystem::TimeoutConfig kTestTimeoutConfig(5, 1, 10, 20, 30); +// Default (empty) constraint config +static std::unordered_set* kAllowedLocationsDefault = + new std::unordered_set(); +// Constraint config if bucket location constraint is turned on, with no +// custom list +static std::unordered_set* kAllowedLocationsAuto = + new std::unordered_set({"auto"}); class FakeAuthProvider : public AuthProvider { public: @@ -33,6 +40,14 @@ class FakeAuthProvider : public AuthProvider { } }; +class FakeZoneProvider : public ZoneProvider { + public: + Status GetZone(string* zone) override { + *zone = "us-east1-b"; + return Status::OK(); + } +}; + TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) { std::vector requests( {new FakeHttpRequest( @@ -47,15 +62,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) { "Range: 6-11\n" "Timeouts: 5 1 20\n", "6789")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -74,6 +90,118 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) { EXPECT_EQ("6789", result); } +TEST(GcsFileSystemTest, + NewRandomAccessFile_WithLocationConstraintInSameLocation) { + std::vector requests({new FakeHttpRequest( + "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", + R"( + { + "location":"us-east1" + })")}); + + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, + 0 /* initial retry delay */, kTestTimeoutConfig, + *kAllowedLocationsAuto, nullptr /* gcs additional header */); + + std::unique_ptr file; + TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); +} + +TEST(GcsFileSystemTest, NewRandomAccessFile_WithLocationConstraintCaching) { + std::vector requests( + {new FakeHttpRequest( + "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", + R"( + { + "location":"us-east1" + })"), + new FakeHttpRequest( + "Uri: https://www.googleapis.com/storage/v1/b/anotherbucket\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", + R"( + { + "location":"us-east1" + })"), + new FakeHttpRequest( + "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", + R"( + { + "location":"us-east1" + })")}); + + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, + 0 /* initial retry delay */, kTestTimeoutConfig, + *kAllowedLocationsAuto, nullptr /* gcs additional header */); + + std::unique_ptr file; + + string bucket = "gs://bucket/random_access.txt"; + string another_bucket = "gs://anotherbucket/random_access.txt"; + // Multiple calls should only cause one request to the location api. + TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file)); + TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file)); + + // A new bucket should have one cache miss + TF_EXPECT_OK(fs.NewRandomAccessFile(another_bucket, &file)); + // And then future calls to both should be cached + TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file)); + TF_EXPECT_OK(fs.NewRandomAccessFile(another_bucket, &file)); + + // Trigger a flush, should then require one more call + fs.FlushCaches(); + TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file)); +} + +TEST(GcsFileSystemTest, + NewRandomAccessFile_WithLocationConstraintInDifferentLocation) { + std::vector requests({new FakeHttpRequest( + "Uri: https://www.googleapis.com/storage/v1/b/bucket\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", + R"( + { + "location":"barfoo" + })")}); + + GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 0 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, + 0 /* initial retry delay */, kTestTimeoutConfig, + *kAllowedLocationsAuto, nullptr /* gcs additional header */); + + std::unique_ptr file; + EXPECT_EQ(tensorflow::errors::FailedPrecondition( + "Bucket 'bucket' is in 'barfoo' location, allowed locations " + "are: (us-east1)."), + fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); +} + TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_DifferentN) { std::vector requests( {new FakeHttpRequest( @@ -88,15 +216,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_DifferentN) { "Range: 3-12\n" "Timeouts: 5 1 20\n", "3456789")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -151,11 +280,12 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, + std::unique_ptr(new FakeZoneProvider), 9 /* block size */, + 18 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); char scratch[100]; StringPiece result; @@ -239,11 +369,12 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_Flush) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, + std::unique_ptr(new FakeZoneProvider), 9 /* block size */, + 18 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); char scratch[100]; StringPiece result; @@ -287,11 +418,13 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_MaxStaleness) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 8 /* block size */, 16 /* max bytes */, 3600 /* max staleness */, + std::unique_ptr(new FakeZoneProvider), 8 /* block size */, + 16 /* max bytes */, 3600 /* max staleness */, 3600 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); char scratch[100]; StringPiece result; // There should only be two HTTP requests issued to GCS even though we iterate @@ -356,11 +489,12 @@ TEST(GcsFileSystemTest, std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, + std::unique_ptr(new FakeZoneProvider), 9 /* block size */, + 18 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); @@ -383,11 +517,13 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoObjectName) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* read ahead bytes */, 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -411,15 +547,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_InconsistentRead) { "012")}); // Set stat_cache_max_age to 1000s so that StatCache could work. - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 1e3 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 1e3 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // Stat the file first so that the file stats are cached. FileStatistics stat; @@ -481,11 +618,12 @@ TEST(GcsFileSystemTest, NewWritableFile) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 8 /* block size */, 8 /* max bytes */, 0 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, + std::unique_ptr(new FakeZoneProvider), 8 /* block size */, + 8 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // Read from the file first, to fill the block cache. std::unique_ptr rfile; @@ -565,15 +703,16 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) { "Timeouts: 5 1 30\n" "Put body: t2\n", "")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); @@ -638,11 +777,13 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceedsOnGetStatus) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 8 /* block size */, 8 /* max bytes */, 3600 /* max staleness */, + std::unique_ptr(new FakeZoneProvider), 8 /* block size */, + 8 /* max bytes */, 3600 /* max staleness */, 3600 /* stat cache max age */, 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // Pull the file's first block into the cache. This will trigger the first // HTTP request to GCS. std::unique_ptr rfile; @@ -719,15 +860,16 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) { "Timeouts: 5 1 30\n" "Put body: content1,content2\n", "")); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 2 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 2 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); @@ -776,15 +918,16 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) { "Timeouts: 5 1 30\n" "Put body: content1,content2\n", "")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); @@ -805,15 +948,16 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) { TEST(GcsFileSystemTest, NewWritableFile_NoObjectName) { std::vector requests; - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -866,11 +1010,12 @@ TEST(GcsFileSystemTest, NewAppendableFile) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 32 /* block size */, 32 /* max bytes */, 0 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, + std::unique_ptr(new FakeZoneProvider), 32 /* block size */, + 32 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // Create an appendable file. This should read the file from GCS, and pull its // contents into the block cache. @@ -896,15 +1041,16 @@ TEST(GcsFileSystemTest, NewAppendableFile) { TEST(GcsFileSystemTest, NewAppendableFile_NoObjectName) { std::vector requests; - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -929,15 +1075,16 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) { "Range: 0-", content.size() - 1, "\n", "Timeouts: 5 1 20\n"), content)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr region; TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile( @@ -949,15 +1096,16 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) { TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile_NoObjectName) { std::vector requests; - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr region; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -972,15 +1120,16 @@ TEST(GcsFileSystemTest, FileExists_YesAsObject) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt")); } @@ -1001,15 +1150,16 @@ TEST(GcsFileSystemTest, FileExists_YesAsFolder) { "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/subfolder/\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder")); } @@ -1026,15 +1176,16 @@ TEST(GcsFileSystemTest, FileExists_YesAsBucket) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{\"size\": \"100\"}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.FileExists("gs://bucket1")); TF_EXPECT_OK(fs.FileExists("gs://bucket1/")); @@ -1055,16 +1206,17 @@ TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{\"items\": []}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); - + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); + EXPECT_EQ(errors::Code::NOT_FOUND, fs.FileExists("gs://bucket/path/file1.txt").code()); } @@ -1081,15 +1233,16 @@ TEST(GcsFileSystemTest, FileExists_NotAsBucket) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, fs.FileExists("gs://bucket2/").code()); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -1123,11 +1276,12 @@ TEST(GcsFileSystemTest, FileExists_StatCache) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // The stat cache will ensure that repeated lookups don't trigger additional // HTTP requests. @@ -1149,11 +1303,12 @@ TEST(GcsFileSystemTest, FileExists_DirectoryMark) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.FileExists("gs://bucket/dir/")); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/dir/")); @@ -1167,15 +1322,16 @@ TEST(GcsFileSystemTest, GetChildren_NoItems) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{\"prefixes\": [\"path/subpath/\"]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -1194,15 +1350,16 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/file3.txt\" }]," "\"prefixes\": [\"path/subpath/\"]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -1222,15 +1379,16 @@ TEST(GcsFileSystemTest, GetChildren_SelfDirectoryMarker) { " { \"name\": \"path/\" }," " { \"name\": \"path/file3.txt\" }]," "\"prefixes\": [\"path/subpath/\"]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -1249,15 +1407,16 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles_NoSlash) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/file3.txt\" }]," "\"prefixes\": [\"path/subpath/\"]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children)); @@ -1273,15 +1432,16 @@ TEST(GcsFileSystemTest, GetChildren_Root) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket-a-b-c", &children)); @@ -1297,15 +1457,16 @@ TEST(GcsFileSystemTest, GetChildren_Empty) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); @@ -1337,15 +1498,16 @@ TEST(GcsFileSystemTest, GetChildren_Pagination) { " { \"name\": \"path/file4.txt\" }," " { \"name\": \"path/file5.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector children; TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children)); @@ -1363,15 +1525,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_NoWildcard) { "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/subpath/file2.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector result; TF_EXPECT_OK( @@ -1390,15 +1553,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_BucketAndWildcard) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," " { \"name\": \"path/file3.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", &result)); @@ -1418,15 +1582,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_Matches) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," " { \"name\": \"path/file3.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file2.txt", &result)); @@ -1443,15 +1608,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_SelfDirectoryMarker) { "{\"items\": [ " " { \"name\": \"path/\" }," " { \"name\": \"path/file3.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", &result)); @@ -1468,15 +1634,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," " { \"name\": \"path/file3.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector result; TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file3.txt", &result)); @@ -1485,15 +1652,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) { TEST(GcsFileSystemTest, GetMatchingPaths_OnlyWildcard) { std::vector requests; - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::vector result; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -1518,15 +1686,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_Cache) { " { \"name\": \"path/file1.txt\" }," " { \"name\": \"path/subpath/file2.txt\" }," " { \"name\": \"path/file3.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 3600 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 3600 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // Repeated calls to fs.GetMatchingPaths on these patterns should not lead to // any additional HTTP requests to GCS. @@ -1560,15 +1729,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_Cache_Flush) { "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/subpath/file2.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 3600 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 3600 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // This loop should trigger the first HTTP request to GCS. for (int i = 0; i < 10; i++) { @@ -1627,11 +1797,12 @@ TEST(GcsFileSystemTest, DeleteFile) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 16 /* block size */, 16 /* max bytes */, 0 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, + std::unique_ptr(new FakeZoneProvider), 16 /* block size */, + 16 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // Do an initial read of the file to load its contents into the block cache. char scratch[100]; @@ -1650,15 +1821,16 @@ TEST(GcsFileSystemTest, DeleteFile) { TEST(GcsFileSystemTest, DeleteFile_NoObjectName) { std::vector requests; - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, fs.DeleteFile("gs://bucket/").code()); @@ -1696,11 +1868,12 @@ TEST(GcsFileSystemTest, DeleteFile_StatCacheRemoved) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 16 /* block size */, 16 /* max bytes */, 0 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, + std::unique_ptr(new FakeZoneProvider), 16 /* block size */, + 16 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // Stats the file first so the stat is cached. FileStatistics stat_before_deletion; @@ -1721,15 +1894,16 @@ TEST(GcsFileSystemTest, DeleteDir_Empty) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/")); } @@ -1749,15 +1923,16 @@ TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) { "Timeouts: 5 1 10\n" "Delete: yes\n", "")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/")); } @@ -1768,15 +1943,16 @@ TEST(GcsFileSystemTest, DeleteDir_BucketOnly) { "name%2CnextPageToken&maxResults=2\nAuth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.DeleteDir("gs://bucket")); } @@ -1789,15 +1965,16 @@ TEST(GcsFileSystemTest, DeleteDir_NonEmpty) { "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"path/file1.txt\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); EXPECT_EQ(error::Code::FAILED_PRECONDITION, fs.DeleteDir("gs://bucket/path/").code()); @@ -1811,15 +1988,16 @@ TEST(GcsFileSystemTest, GetFileSize) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); uint64 size; TF_EXPECT_OK(fs.GetFileSize("gs://bucket/file.txt", &size)); @@ -1828,15 +2006,16 @@ TEST(GcsFileSystemTest, GetFileSize) { TEST(GcsFileSystemTest, GetFileSize_NoObjectName) { std::vector requests; - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); uint64 size; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, @@ -1913,15 +2092,16 @@ TEST(GcsFileSystemTest, RenameFile_Folder) { "Timeouts: 5 1 10\n" "Delete: yes\n", "")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.RenameFile("gs://bucket/path1", "gs://bucket/path2/")); } @@ -2008,11 +2188,12 @@ TEST(GcsFileSystemTest, RenameFile_Object) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 16 /* block size */, 64 /* max bytes */, 0 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, + std::unique_ptr(new FakeZoneProvider), 16 /* block size */, + 64 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // Do an initial read of the source and destination files to load their // contents into the block cache. char scratch[100]; @@ -2088,11 +2269,12 @@ TEST(GcsFileSystemTest, RenameFile_Object_FlushTargetStatCache) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // Do an initial stat of the destination file to load their contents into the // stat cache. FileStatistics stat_before_renaming; @@ -2150,15 +2332,16 @@ TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) { "Timeouts: 5 1 10\n" "Delete: yes\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK( fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt")); @@ -2186,20 +2369,21 @@ TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) { // Copying to the new location. new FakeHttpRequest( "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" - "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n" - "Auth Token: fake_token\n" - "Post: yes\n" - "Timeouts: 5 1 10\n", - "{\"done\": false}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n" + "Auth Token: fake_token\n" + "Post: yes\n" + "Timeouts: 5 1 10\n", + "{\"done\": false}")}); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); EXPECT_EQ( errors::Code::UNIMPLEMENTED, @@ -2215,15 +2399,16 @@ TEST(GcsFileSystemTest, Stat_Object) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat)); @@ -2248,15 +2433,16 @@ TEST(GcsFileSystemTest, Stat_Folder) { "Timeouts: 5 1 10\n", "{\"items\": [ " " { \"name\": \"subfolder/\" }]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", &stat)); @@ -2280,15 +2466,16 @@ TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); FileStatistics stat; EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/path", &stat).code()); @@ -2300,15 +2487,16 @@ TEST(GcsFileSystemTest, Stat_Bucket) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/", &stat)); @@ -2323,15 +2511,16 @@ TEST(GcsFileSystemTest, Stat_BucketNotFound) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); FileStatistics stat; EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/", &stat).code()); @@ -2364,11 +2553,12 @@ TEST(GcsFileSystemTest, Stat_Cache) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // Repeated calls to fs.Stat on these paths should not lead to any additional // HTTP requests to GCS. @@ -2405,11 +2595,12 @@ TEST(GcsFileSystemTest, Stat_Cache_Flush) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 3600 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, - kTestTimeoutConfig, nullptr /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); // There should be a single HTTP request to GCS for fs.Stat in this loop. for (int i = 0; i < 10; i++) { FileStatistics stat; @@ -2437,15 +2628,16 @@ TEST(GcsFileSystemTest, Stat_FilenameEndingWithSlash) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"5\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); FileStatistics stat; TF_EXPECT_OK(fs.Stat("gs://bucket/dir/", &stat)); @@ -2468,15 +2660,16 @@ TEST(GcsFileSystemTest, IsDirectory_NotFound) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/file.txt").code()); @@ -2498,15 +2691,16 @@ TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); EXPECT_EQ(error::Code::FAILED_PRECONDITION, fs.IsDirectory("gs://bucket/file.txt").code()); @@ -2528,15 +2722,16 @@ TEST(GcsFileSystemTest, IsDirectory_Yes) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{\"items\": [{\"name\": \"subfolder/\"}]}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder")); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder/")); @@ -2554,15 +2749,16 @@ TEST(GcsFileSystemTest, IsDirectory_Bucket) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.IsDirectory("gs://bucket")); TF_EXPECT_OK(fs.IsDirectory("gs://bucket/")); @@ -2574,15 +2770,16 @@ TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/").code()); } @@ -2615,15 +2812,16 @@ TEST(GcsFileSystemTest, CreateDir_Folder) { "Timeouts: 5 1 30\n" "Put body: \n", "")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath")); TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath/")); @@ -2641,15 +2839,16 @@ TEST(GcsFileSystemTest, CreateDir_Bucket) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TF_EXPECT_OK(fs.CreateDir("gs://bucket/")); TF_EXPECT_OK(fs.CreateDir("gs://bucket")); @@ -2712,15 +2911,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_Ok) { "Timeouts: 5 1 10\n" "Delete: yes\n", "")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); int64 undeleted_files, undeleted_dirs; TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files, @@ -2804,15 +3004,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) { "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); int64 undeleted_files, undeleted_dirs; TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files, @@ -2838,15 +3039,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) { "Auth Token: fake_token\n" "Timeouts: 5 1 10\n", "", errors::NotFound("404"), 404)}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay*/, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); int64 undeleted_files, undeleted_dirs; EXPECT_EQ(error::Code::NOT_FOUND, @@ -2857,6 +3059,29 @@ TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) { EXPECT_EQ(1, undeleted_dirs); } +TEST(GcsFileSystemTest, NoConstraintsEnvironmentVariableTest) { + unsetenv("GCS_ALLOWED_BUCKET_LOCATIONS"); + // No constraints + GcsFileSystem fs1; + EXPECT_EQ(*kAllowedLocationsDefault, fs1.allowed_locations()); + + // Cover cache initialization code, any uninitialized cache will cause this to + // fail + fs1.FlushCaches(); +} + +TEST(GcsFileSystemTest, BucketLocationConstraintEnvironmentVariableTest) { + unsetenv("GCS_ALLOWED_BUCKET_LOCATIONS"); + setenv("GCS_ALLOWED_BUCKET_LOCATIONS", "auto", 1); + GcsFileSystem fs1; + EXPECT_EQ(*kAllowedLocationsAuto, fs1.allowed_locations()); + + setenv("GCS_ALLOWED_BUCKET_LOCATIONS", "custom,list", 1); + GcsFileSystem fs2; + EXPECT_EQ(std::unordered_set({"custom", "list"}), + fs2.allowed_locations()); +} + TEST(GcsFileSystemTest, AdditionalRequestHeaderTest) { GcsFileSystem fs1; EXPECT_EQ("", fs1.additional_header_name()); @@ -2902,11 +3127,12 @@ TEST(GcsFileSystemTest, AdditionalRequestHeaderTest) { std::unique_ptr(new FakeAuthProvider), std::unique_ptr( new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, 0 /* matching paths cache max entries */, 0 /* initial retry delay */, - kTestTimeoutConfig, add_header /* gcs additional header */); + kTestTimeoutConfig, *kAllowedLocationsDefault, + add_header /* gcs additional header */); std::unique_ptr request; TF_EXPECT_OK(fs7.CreateHttpRequest(&request)); @@ -2973,15 +3199,16 @@ TEST(GcsFileSystemTest, CreateHttpRequest) { "Auth Token: fake_token\n" "Header Hello: world\n", "{}")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); std::unique_ptr request; TF_EXPECT_OK(fs.CreateHttpRequest(&request)); @@ -3035,15 +3262,16 @@ TEST(GcsFileSystemTest, Stat_StatsRecording) { "Timeouts: 5 1 10\n", strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\"," "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TestGcsStats stats; fs.SetStats(&stats); @@ -3061,15 +3289,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_StatsRecording) { "Range: 0-5\n" "Timeouts: 5 1 20\n", "012345")}); - GcsFileSystem fs(std::unique_ptr(new FakeAuthProvider), - std::unique_ptr( - new FakeHttpRequestFactory(&requests)), - 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, - 0 /* stat cache max age */, 0 /* stat cache max entries */, - 0 /* matching paths cache max age */, - 0 /* matching paths cache max entries */, - 0 /* initial retry delay */, kTestTimeoutConfig, - nullptr /* gcs additional header */); + GcsFileSystem fs( + std::unique_ptr(new FakeAuthProvider), + std::unique_ptr( + new FakeHttpRequestFactory(&requests)), + std::unique_ptr(new FakeZoneProvider), 0 /* block size */, + 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */, + 0 /* stat cache max entries */, 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay */, + kTestTimeoutConfig, *kAllowedLocationsDefault, + nullptr /* gcs additional header */); TestGcsStats stats; fs.SetStats(&stats); -- cgit v1.2.3