aboutsummaryrefslogtreecommitdiffhomepage
path: root/tensorflow/core/platform
diff options
context:
space:
mode:
authorGravatar Michael Banfield <micban@google.com>2018-08-02 13:13:48 -0700
committerGravatar TensorFlower Gardener <gardener@tensorflow.org>2018-08-02 13:17:43 -0700
commit8ec17cff881923fe3ff26b7de812cf0b2d21cd07 (patch)
tree6b05826db3c7d3a635d75f0504f9b0de3d8bff3f /tensorflow/core/platform
parentbcb5a132684424ff678e9d64fc291f49ce7fcc4c (diff)
Add a optional bucket location check to the GCS Filesystem.
PiperOrigin-RevId: 207152562
Diffstat (limited to 'tensorflow/core/platform')
-rw-r--r--tensorflow/core/platform/cloud/BUILD1
-rw-r--r--tensorflow/core/platform/cloud/gcs_file_system.cc110
-rw-r--r--tensorflow/core/platform/cloud/gcs_file_system.h37
-rw-r--r--tensorflow/core/platform/cloud/gcs_file_system_test.cc1413
4 files changed, 959 insertions, 602 deletions
diff --git a/tensorflow/core/platform/cloud/BUILD b/tensorflow/core/platform/cloud/BUILD
index 549996aaf8..647a797b82 100644
--- a/tensorflow/core/platform/cloud/BUILD
+++ b/tensorflow/core/platform/cloud/BUILD
@@ -74,6 +74,7 @@ cc_library(
visibility = ["//visibility:public"],
deps = [
":compute_engine_metadata_client",
+ ":compute_engine_zone_provider",
":curl_http_request",
":expiring_lru_cache",
":file_block_cache",
diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc
index 2e8d13acd5..67c872ac67 100644
--- a/tensorflow/core/platform/cloud/gcs_file_system.cc
+++ b/tensorflow/core/platform/cloud/gcs_file_system.cc
@@ -57,6 +57,7 @@ constexpr char kGcsUriBase[] = "https://www.googleapis.com/storage/v1/";
constexpr char kGcsUploadUriBase[] =
"https://www.googleapis.com/upload/storage/v1/";
constexpr char kStorageHost[] = "storage.googleapis.com";
+constexpr char kBucketMetadataLocationKey[] = "location";
constexpr size_t kReadAppendableFileBufferSize = 1024 * 1024; // In bytes.
constexpr int kGetChildrenDefaultPageSize = 1000;
// The HTTP response code "308 Resume Incomplete".
@@ -98,6 +99,11 @@ constexpr uint64 kMatchingPathsCacheDefaultMaxAge = 0;
constexpr char kMatchingPathsCacheMaxEntries[] =
"GCS_MATCHING_PATHS_CACHE_MAX_ENTRIES";
constexpr size_t kMatchingPathsCacheDefaultMaxEntries = 1024;
+// Number of bucket locations cached, most workloads wont touch more than one
+// bucket so this limit is set fairly low
+constexpr size_t kBucketLocationCacheMaxEntries = 10;
+// ExpiringLRUCache doesnt support any "cache forever" option
+constexpr size_t kCacheNeverExpire = std::numeric_limits<uint64>::max();
// The file statistics returned by Stat() for directories.
const FileStatistics DIRECTORY_STAT(0, 0, true);
// Some environments exhibit unreliable DNS resolution. Set this environment
@@ -131,6 +137,14 @@ constexpr char kTokensPerRequest[] = "GCS_TOKENS_PER_REQUEST";
// The environment variable to configure the initial tokens (format: <int64>)
constexpr char kInitialTokens[] = "GCS_INITIAL_TOKENS";
+// The environment variable to customize which GCS bucket locations are allowed,
+// if the list is empty defaults to using the region of the zone (format, comma
+// delimited list). Requires 'storage.buckets.get' permission.
+constexpr char kAllowedBucketLocations[] = "GCS_ALLOWED_BUCKET_LOCATIONS";
+// When this value is passed as an allowed location detects the zone tensorflow
+// is running in and restricts to buckets in that region.
+constexpr char kDetectZoneSentinalValue[] = "auto";
+
// TODO: DO NOT use a hardcoded path
Status GetTmpFilename(string* filename) {
#ifndef _WIN32
@@ -603,6 +617,19 @@ bool StringPieceIdentity(StringPiece str, StringPiece* value) {
return true;
}
+/// \brief Utility function to split a comma delimited list of strings to an
+/// unordered set
+bool SplitByCommaToSet(StringPiece list, std::unordered_set<string>* set) {
+ std::vector<string> vector = str_util::Split(list, ",");
+ *set = std::unordered_set<string>(vector.begin(), vector.end());
+ return true;
+}
+
+// \brief Convert Compute Engine zone to region
+string ZoneToRegion(string* zone) {
+ return zone->substr(0, zone->find_last_of('-'));
+}
+
} // namespace
GcsFileSystem::GcsFileSystem() {
@@ -616,6 +643,8 @@ GcsFileSystem::GcsFileSystem() {
std::make_shared<ComputeEngineMetadataClient>(http_request_factory_);
auth_provider_ = std::unique_ptr<AuthProvider>(
new GoogleAuthProvider(compute_engine_metadata_client_));
+ zone_provider_ = std::unique_ptr<ZoneProvider>(
+ new ComputeEngineZoneProvider(compute_engine_metadata_client_));
// Apply the sys env override for the readahead buffer size if it's provided.
if (GetEnvVar(kReadaheadBufferSize, strings::safe_strtou64, &value)) {
@@ -666,6 +695,9 @@ GcsFileSystem::GcsFileSystem() {
matching_paths_cache_.reset(new ExpiringLRUCache<std::vector<string>>(
matching_paths_cache_max_age, matching_paths_cache_max_entries));
+ bucket_location_cache_.reset(new ExpiringLRUCache<string>(
+ kCacheNeverExpire, kBucketLocationCacheMaxEntries));
+
int64 resolve_frequency_secs;
if (GetEnvVar(kResolveCacheSecs, strings::safe_strto64,
&resolve_frequency_secs)) {
@@ -745,24 +777,30 @@ GcsFileSystem::GcsFileSystem() {
}
throttle_.SetConfig(config);
}
+
+ GetEnvVar(kAllowedBucketLocations, SplitByCommaToSet, &allowed_locations_);
}
GcsFileSystem::GcsFileSystem(
std::unique_ptr<AuthProvider> auth_provider,
std::unique_ptr<HttpRequest::Factory> http_request_factory,
- size_t block_size, size_t max_bytes, uint64 max_staleness,
- uint64 stat_cache_max_age, size_t stat_cache_max_entries,
- uint64 matching_paths_cache_max_age,
+ std::unique_ptr<ZoneProvider> zone_provider, size_t block_size,
+ size_t max_bytes, uint64 max_staleness, uint64 stat_cache_max_age,
+ size_t stat_cache_max_entries, uint64 matching_paths_cache_max_age,
size_t matching_paths_cache_max_entries, int64 initial_retry_delay_usec,
- TimeoutConfig timeouts,
+ TimeoutConfig timeouts, const std::unordered_set<string>& allowed_locations,
std::pair<const string, const string>* additional_header)
: auth_provider_(std::move(auth_provider)),
http_request_factory_(std::move(http_request_factory)),
+ zone_provider_(std::move(zone_provider)),
file_block_cache_(
MakeFileBlockCache(block_size, max_bytes, max_staleness)),
stat_cache_(new StatCache(stat_cache_max_age, stat_cache_max_entries)),
matching_paths_cache_(new MatchingPathsCache(
matching_paths_cache_max_age, matching_paths_cache_max_entries)),
+ bucket_location_cache_(new BucketLocationCache(
+ kCacheNeverExpire, kBucketLocationCacheMaxEntries)),
+ allowed_locations_(allowed_locations),
timeouts_(timeouts),
initial_retry_delay_usec_(initial_retry_delay_usec),
additional_header_(additional_header) {}
@@ -771,6 +809,7 @@ Status GcsFileSystem::NewRandomAccessFile(
const string& fname, std::unique_ptr<RandomAccessFile>* result) {
string bucket, object;
TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object));
+ TF_RETURN_IF_ERROR(CheckBucketLocationConstraint(bucket));
result->reset(new GcsRandomAccessFile(fname, [this, bucket, object](
const string& fname,
uint64 offset, size_t n,
@@ -1072,11 +1111,7 @@ Status GcsFileSystem::StatForObject(const string& fname, const string& bucket,
}
Status GcsFileSystem::BucketExists(const string& bucket, bool* result) {
- std::unique_ptr<HttpRequest> request;
- TF_RETURN_IF_ERROR(CreateHttpRequest(&request));
- request->SetUri(strings::StrCat(kGcsUriBase, "b/", bucket));
- request->SetTimeouts(timeouts_.connect, timeouts_.idle, timeouts_.metadata);
- const Status status = request->Send();
+ const Status status = GetBucketMetadata(bucket, nullptr);
switch (status.code()) {
case errors::Code::OK:
*result = true;
@@ -1089,6 +1124,62 @@ Status GcsFileSystem::BucketExists(const string& bucket, bool* result) {
}
}
+Status GcsFileSystem::CheckBucketLocationConstraint(const string& bucket) {
+ if (allowed_locations_.empty()) {
+ return Status::OK();
+ }
+
+ // Avoid calling external API's in the constructor
+ if (allowed_locations_.erase(kDetectZoneSentinalValue) == 1) {
+ string zone;
+ TF_RETURN_IF_ERROR(zone_provider_->GetZone(&zone));
+ allowed_locations_.insert(ZoneToRegion(&zone));
+ }
+
+ string location;
+ TF_RETURN_IF_ERROR(GetBucketLocation(bucket, &location));
+ if (allowed_locations_.find(location) != allowed_locations_.end()) {
+ return Status::OK();
+ }
+
+ return errors::FailedPrecondition(strings::Printf(
+ "Bucket '%s' is in '%s' location, allowed locations are: (%s).",
+ bucket.c_str(), location.c_str(),
+ str_util::Join(allowed_locations_, ", ").c_str()));
+}
+
+Status GcsFileSystem::GetBucketLocation(const string& bucket,
+ string* location) {
+ auto compute_func = [this](const string& bucket, string* location) {
+ std::vector<char> result_buffer;
+ Status status = GetBucketMetadata(bucket, &result_buffer);
+ Json::Value result;
+ TF_RETURN_IF_ERROR(ParseJson(result_buffer, &result));
+ TF_RETURN_IF_ERROR(
+ GetStringValue(result, kBucketMetadataLocationKey, location));
+ return Status::OK();
+ };
+
+ TF_RETURN_IF_ERROR(
+ bucket_location_cache_->LookupOrCompute(bucket, location, compute_func));
+
+ return Status::OK();
+}
+
+Status GcsFileSystem::GetBucketMetadata(const string& bucket,
+ std::vector<char>* result_buffer) {
+ std::unique_ptr<HttpRequest> request;
+ TF_RETURN_IF_ERROR(CreateHttpRequest(&request));
+ request->SetUri(strings::StrCat(kGcsUriBase, "b/", bucket));
+
+ if (result_buffer != nullptr) {
+ request->SetResultBuffer(result_buffer);
+ }
+
+ request->SetTimeouts(timeouts_.connect, timeouts_.idle, timeouts_.metadata);
+ return request->Send();
+}
+
Status GcsFileSystem::FolderExists(const string& dirname, bool* result) {
StatCache::ComputeFunc compute_func = [this](const string& dirname,
GcsFileStat* stat) {
@@ -1514,6 +1605,7 @@ void GcsFileSystem::FlushCaches() {
file_block_cache_->Flush();
stat_cache_->Clear();
matching_paths_cache_->Clear();
+ bucket_location_cache_->Clear();
}
void GcsFileSystem::SetStats(GcsStatsInterface* stats) {
diff --git a/tensorflow/core/platform/cloud/gcs_file_system.h b/tensorflow/core/platform/cloud/gcs_file_system.h
index a0372286f5..71db707687 100644
--- a/tensorflow/core/platform/cloud/gcs_file_system.h
+++ b/tensorflow/core/platform/cloud/gcs_file_system.h
@@ -23,6 +23,7 @@ limitations under the License.
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/platform/cloud/auth_provider.h"
#include "tensorflow/core/platform/cloud/compute_engine_metadata_client.h"
+#include "tensorflow/core/platform/cloud/compute_engine_zone_provider.h"
#include "tensorflow/core/platform/cloud/expiring_lru_cache.h"
#include "tensorflow/core/platform/cloud/file_block_cache.h"
#include "tensorflow/core/platform/cloud/gcs_dns_cache.h"
@@ -81,14 +82,19 @@ class GcsFileSystem : public FileSystem {
public:
struct TimeoutConfig;
+ // Main constructor used (via RetryingFileSystem) throughout Tensorflow
GcsFileSystem();
+ // Used mostly for unit testing or use cases which need to customize the
+ // filesystem from defaults
GcsFileSystem(std::unique_ptr<AuthProvider> auth_provider,
std::unique_ptr<HttpRequest::Factory> http_request_factory,
- size_t block_size, size_t max_bytes, uint64 max_staleness,
+ std::unique_ptr<ZoneProvider> zone_provider, size_t block_size,
+ size_t max_bytes, uint64 max_staleness,
uint64 stat_cache_max_age, size_t stat_cache_max_entries,
uint64 matching_paths_cache_max_age,
size_t matching_paths_cache_max_entries,
int64 initial_retry_delay_usec, TimeoutConfig timeouts,
+ const std::unordered_set<string>& allowed_locations,
std::pair<const string, const string>* additional_header);
Status NewRandomAccessFile(
@@ -149,6 +155,9 @@ class GcsFileSystem : public FileSystem {
return file_block_cache_->max_staleness();
}
TimeoutConfig timeouts() const { return timeouts_; }
+ std::unordered_set<string> allowed_locations() const {
+ return allowed_locations_;
+ }
string additional_header_name() const {
return additional_header_ ? additional_header_->first : "";
}
@@ -230,6 +239,27 @@ class GcsFileSystem : public FileSystem {
/// 'result' is set if the function returns OK. 'result' cannot be nullptr.
Status BucketExists(const string& bucket, bool* result);
+ /// \brief Retrieves the GCS bucket location. Returns OK if the location was
+ /// retrieved.
+ ///
+ /// Given a string bucket the GCS bucket metadata API will be called and the
+ /// location string filled with the location of the bucket.
+ ///
+ /// This requires the bucket metadata permission.
+ /// Repeated calls for the same bucket are cached so this function can be
+ /// called frequently without causing an extra API call
+ Status GetBucketLocation(const string& bucket, string* location);
+
+ /// \brief Check if the GCS buckets location is allowed with the current
+ /// constraint configuration
+ Status CheckBucketLocationConstraint(const string& bucket);
+
+ /// \brief Given the input bucket `bucket`, fills `result_buffer` with the
+ /// results of the metadata. Returns OK if the API call succeeds without
+ /// error.
+ Status GetBucketMetadata(const string& bucket,
+ std::vector<char>* result_buffer);
+
/// \brief Checks if the object exists. Returns OK if the check succeeded.
///
/// 'result' is set if the function returns OK. 'result' cannot be nullptr.
@@ -277,6 +307,7 @@ class GcsFileSystem : public FileSystem {
mutex mu_;
std::unique_ptr<AuthProvider> auth_provider_ GUARDED_BY(mu_);
std::shared_ptr<HttpRequest::Factory> http_request_factory_;
+ std::unique_ptr<ZoneProvider> zone_provider_;
// block_cache_lock_ protects the file_block_cache_ pointer (Note that
// FileBlockCache instances are themselves threadsafe).
mutex block_cache_lock_;
@@ -292,6 +323,10 @@ class GcsFileSystem : public FileSystem {
using MatchingPathsCache = ExpiringLRUCache<std::vector<string>>;
std::unique_ptr<MatchingPathsCache> matching_paths_cache_;
+ using BucketLocationCache = ExpiringLRUCache<string>;
+ std::unique_ptr<BucketLocationCache> bucket_location_cache_;
+ std::unordered_set<string> allowed_locations_;
+
TimeoutConfig timeouts_;
GcsStatsInterface* stats_ = nullptr; // Not owned.
diff --git a/tensorflow/core/platform/cloud/gcs_file_system_test.cc b/tensorflow/core/platform/cloud/gcs_file_system_test.cc
index e791ae5a19..ee2b034d74 100644
--- a/tensorflow/core/platform/cloud/gcs_file_system_test.cc
+++ b/tensorflow/core/platform/cloud/gcs_file_system_test.cc
@@ -24,6 +24,13 @@ namespace tensorflow {
namespace {
static GcsFileSystem::TimeoutConfig kTestTimeoutConfig(5, 1, 10, 20, 30);
+// Default (empty) constraint config
+static std::unordered_set<string>* kAllowedLocationsDefault =
+ new std::unordered_set<string>();
+// Constraint config if bucket location constraint is turned on, with no
+// custom list
+static std::unordered_set<string>* kAllowedLocationsAuto =
+ new std::unordered_set<string>({"auto"});
class FakeAuthProvider : public AuthProvider {
public:
@@ -33,6 +40,14 @@ class FakeAuthProvider : public AuthProvider {
}
};
+class FakeZoneProvider : public ZoneProvider {
+ public:
+ Status GetZone(string* zone) override {
+ *zone = "us-east1-b";
+ return Status::OK();
+ }
+};
+
TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) {
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
@@ -47,15 +62,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) {
"Range: 6-11\n"
"Timeouts: 5 1 20\n",
"6789")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::unique_ptr<RandomAccessFile> file;
TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
@@ -74,6 +90,118 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) {
EXPECT_EQ("6789", result);
}
+TEST(GcsFileSystemTest,
+ NewRandomAccessFile_WithLocationConstraintInSameLocation) {
+ std::vector<HttpRequest*> requests({new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ R"(
+ {
+ "location":"us-east1"
+ })")});
+
+ GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* stat cache max age */, 0 /* stat cache max entries */,
+ 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */,
+ 0 /* initial retry delay */, kTestTimeoutConfig,
+ *kAllowedLocationsAuto, nullptr /* gcs additional header */);
+
+ std::unique_ptr<RandomAccessFile> file;
+ TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
+}
+
+TEST(GcsFileSystemTest, NewRandomAccessFile_WithLocationConstraintCaching) {
+ std::vector<HttpRequest*> requests(
+ {new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ R"(
+ {
+ "location":"us-east1"
+ })"),
+ new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/anotherbucket\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ R"(
+ {
+ "location":"us-east1"
+ })"),
+ new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ R"(
+ {
+ "location":"us-east1"
+ })")});
+
+ GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* stat cache max age */, 0 /* stat cache max entries */,
+ 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */,
+ 0 /* initial retry delay */, kTestTimeoutConfig,
+ *kAllowedLocationsAuto, nullptr /* gcs additional header */);
+
+ std::unique_ptr<RandomAccessFile> file;
+
+ string bucket = "gs://bucket/random_access.txt";
+ string another_bucket = "gs://anotherbucket/random_access.txt";
+ // Multiple calls should only cause one request to the location api.
+ TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file));
+ TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file));
+
+ // A new bucket should have one cache miss
+ TF_EXPECT_OK(fs.NewRandomAccessFile(another_bucket, &file));
+ // And then future calls to both should be cached
+ TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file));
+ TF_EXPECT_OK(fs.NewRandomAccessFile(another_bucket, &file));
+
+ // Trigger a flush, should then require one more call
+ fs.FlushCaches();
+ TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file));
+}
+
+TEST(GcsFileSystemTest,
+ NewRandomAccessFile_WithLocationConstraintInDifferentLocation) {
+ std::vector<HttpRequest*> requests({new FakeHttpRequest(
+ "Uri: https://www.googleapis.com/storage/v1/b/bucket\n"
+ "Auth Token: fake_token\n"
+ "Timeouts: 5 1 10\n",
+ R"(
+ {
+ "location":"barfoo"
+ })")});
+
+ GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* stat cache max age */, 0 /* stat cache max entries */,
+ 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */,
+ 0 /* initial retry delay */, kTestTimeoutConfig,
+ *kAllowedLocationsAuto, nullptr /* gcs additional header */);
+
+ std::unique_ptr<RandomAccessFile> file;
+ EXPECT_EQ(tensorflow::errors::FailedPrecondition(
+ "Bucket 'bucket' is in 'barfoo' location, allowed locations "
+ "are: (us-east1)."),
+ fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
+}
+
TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_DifferentN) {
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
@@ -88,15 +216,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_DifferentN) {
"Range: 3-12\n"
"Timeouts: 5 1 20\n",
"3456789")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::unique_ptr<RandomAccessFile> file;
TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
@@ -151,11 +280,12 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */,
- 3600 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 9 /* block size */,
+ 18 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
char scratch[100];
StringPiece result;
@@ -239,11 +369,12 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_Flush) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */,
- 3600 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 9 /* block size */,
+ 18 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
char scratch[100];
StringPiece result;
@@ -287,11 +418,13 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_MaxStaleness) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 8 /* block size */, 16 /* max bytes */, 3600 /* max staleness */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 8 /* block size */,
+ 16 /* max bytes */, 3600 /* max staleness */,
3600 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
char scratch[100];
StringPiece result;
// There should only be two HTTP requests issued to GCS even though we iterate
@@ -356,11 +489,12 @@ TEST(GcsFileSystemTest,
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 9 /* block size */, 18 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 9 /* block size */,
+ 18 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::unique_ptr<RandomAccessFile> file;
TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
@@ -383,11 +517,13 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoObjectName) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider),
0 /* read ahead bytes */, 0 /* max bytes */, 0 /* max staleness */,
0 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::unique_ptr<RandomAccessFile> file;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -411,15 +547,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_InconsistentRead) {
"012")});
// Set stat_cache_max_age to 1000s so that StatCache could work.
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 1e3 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 1e3 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
// Stat the file first so that the file stats are cached.
FileStatistics stat;
@@ -481,11 +618,12 @@ TEST(GcsFileSystemTest, NewWritableFile) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 8 /* block size */, 8 /* max bytes */, 0 /* max staleness */,
- 3600 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 8 /* block size */,
+ 8 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
// Read from the file first, to fill the block cache.
std::unique_ptr<RandomAccessFile> rfile;
@@ -565,15 +703,16 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) {
"Timeouts: 5 1 30\n"
"Put body: t2\n",
"")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::unique_ptr<WritableFile> file;
TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file));
@@ -638,11 +777,13 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceedsOnGetStatus) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 8 /* block size */, 8 /* max bytes */, 3600 /* max staleness */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 8 /* block size */,
+ 8 /* max bytes */, 3600 /* max staleness */,
3600 /* stat cache max age */, 0 /* stat cache max entries */,
0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
// Pull the file's first block into the cache. This will trigger the first
// HTTP request to GCS.
std::unique_ptr<RandomAccessFile> rfile;
@@ -719,15 +860,16 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) {
"Timeouts: 5 1 30\n"
"Put body: content1,content2\n",
""));
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 2 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 2 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::unique_ptr<WritableFile> file;
TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file));
@@ -776,15 +918,16 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) {
"Timeouts: 5 1 30\n"
"Put body: content1,content2\n",
"")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::unique_ptr<WritableFile> file;
TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file));
@@ -805,15 +948,16 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) {
TEST(GcsFileSystemTest, NewWritableFile_NoObjectName) {
std::vector<HttpRequest*> requests;
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::unique_ptr<WritableFile> file;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -866,11 +1010,12 @@ TEST(GcsFileSystemTest, NewAppendableFile) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 32 /* block size */, 32 /* max bytes */, 0 /* max staleness */,
- 3600 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 32 /* block size */,
+ 32 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
// Create an appendable file. This should read the file from GCS, and pull its
// contents into the block cache.
@@ -896,15 +1041,16 @@ TEST(GcsFileSystemTest, NewAppendableFile) {
TEST(GcsFileSystemTest, NewAppendableFile_NoObjectName) {
std::vector<HttpRequest*> requests;
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::unique_ptr<WritableFile> file;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -929,15 +1075,16 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) {
"Range: 0-",
content.size() - 1, "\n", "Timeouts: 5 1 20\n"),
content)});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::unique_ptr<ReadOnlyMemoryRegion> region;
TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile(
@@ -949,15 +1096,16 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) {
TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile_NoObjectName) {
std::vector<HttpRequest*> requests;
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::unique_ptr<ReadOnlyMemoryRegion> region;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -972,15 +1120,16 @@ TEST(GcsFileSystemTest, FileExists_YesAsObject) {
"Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt"));
}
@@ -1001,15 +1150,16 @@ TEST(GcsFileSystemTest, FileExists_YesAsFolder) {
"Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/subfolder/\" }]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder"));
}
@@ -1026,15 +1176,16 @@ TEST(GcsFileSystemTest, FileExists_YesAsBucket) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"{\"size\": \"100\"}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
TF_EXPECT_OK(fs.FileExists("gs://bucket1"));
TF_EXPECT_OK(fs.FileExists("gs://bucket1/"));
@@ -1055,15 +1206,16 @@ TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"{\"items\": []}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
EXPECT_EQ(errors::Code::NOT_FOUND,
fs.FileExists("gs://bucket/path/file1.txt").code());
@@ -1081,15 +1233,16 @@ TEST(GcsFileSystemTest, FileExists_NotAsBucket) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404)});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
fs.FileExists("gs://bucket2/").code());
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -1123,11 +1276,12 @@ TEST(GcsFileSystemTest, FileExists_StatCache) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 3600 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
// The stat cache will ensure that repeated lookups don't trigger additional
// HTTP requests.
@@ -1149,11 +1303,12 @@ TEST(GcsFileSystemTest, FileExists_DirectoryMark) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 3600 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
TF_EXPECT_OK(fs.FileExists("gs://bucket/dir/"));
TF_EXPECT_OK(fs.IsDirectory("gs://bucket/dir/"));
@@ -1167,15 +1322,16 @@ TEST(GcsFileSystemTest, GetChildren_NoItems) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"{\"prefixes\": [\"path/subpath/\"]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
@@ -1194,15 +1350,16 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles) {
" { \"name\": \"path/file1.txt\" },"
" { \"name\": \"path/file3.txt\" }],"
"\"prefixes\": [\"path/subpath/\"]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
@@ -1222,15 +1379,16 @@ TEST(GcsFileSystemTest, GetChildren_SelfDirectoryMarker) {
" { \"name\": \"path/\" },"
" { \"name\": \"path/file3.txt\" }],"
"\"prefixes\": [\"path/subpath/\"]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
@@ -1249,15 +1407,16 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles_NoSlash) {
" { \"name\": \"path/file1.txt\" },"
" { \"name\": \"path/file3.txt\" }],"
"\"prefixes\": [\"path/subpath/\"]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children));
@@ -1273,15 +1432,16 @@ TEST(GcsFileSystemTest, GetChildren_Root) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"{}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket-a-b-c", &children));
@@ -1297,15 +1457,16 @@ TEST(GcsFileSystemTest, GetChildren_Empty) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"{}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
@@ -1337,15 +1498,16 @@ TEST(GcsFileSystemTest, GetChildren_Pagination) {
" { \"name\": \"path/file4.txt\" },"
" { \"name\": \"path/file5.txt\" }]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children));
@@ -1363,15 +1525,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_NoWildcard) {
"Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/subpath/file2.txt\" }]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::vector<string> result;
TF_EXPECT_OK(
@@ -1390,15 +1553,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_BucketAndWildcard) {
" { \"name\": \"path/file1.txt\" },"
" { \"name\": \"path/subpath/file2.txt\" },"
" { \"name\": \"path/file3.txt\" }]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::vector<string> result;
TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", &result));
@@ -1418,15 +1582,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_Matches) {
" { \"name\": \"path/file1.txt\" },"
" { \"name\": \"path/subpath/file2.txt\" },"
" { \"name\": \"path/file3.txt\" }]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::vector<string> result;
TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file2.txt", &result));
@@ -1443,15 +1608,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_SelfDirectoryMarker) {
"{\"items\": [ "
" { \"name\": \"path/\" },"
" { \"name\": \"path/file3.txt\" }]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::vector<string> result;
TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", &result));
@@ -1468,15 +1634,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) {
" { \"name\": \"path/file1.txt\" },"
" { \"name\": \"path/subpath/file2.txt\" },"
" { \"name\": \"path/file3.txt\" }]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::vector<string> result;
TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file3.txt", &result));
@@ -1485,15 +1652,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) {
TEST(GcsFileSystemTest, GetMatchingPaths_OnlyWildcard) {
std::vector<HttpRequest*> requests;
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::vector<string> result;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -1518,15 +1686,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_Cache) {
" { \"name\": \"path/file1.txt\" },"
" { \"name\": \"path/subpath/file2.txt\" },"
" { \"name\": \"path/file3.txt\" }]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 3600 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 3600 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
// Repeated calls to fs.GetMatchingPaths on these patterns should not lead to
// any additional HTTP requests to GCS.
@@ -1560,15 +1729,16 @@ TEST(GcsFileSystemTest, GetMatchingPaths_Cache_Flush) {
"Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/subpath/file2.txt\" }]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 3600 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 3600 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
// This loop should trigger the first HTTP request to GCS.
for (int i = 0; i < 10; i++) {
@@ -1627,11 +1797,12 @@ TEST(GcsFileSystemTest, DeleteFile) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 16 /* block size */, 16 /* max bytes */, 0 /* max staleness */,
- 3600 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 16 /* block size */,
+ 16 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
// Do an initial read of the file to load its contents into the block cache.
char scratch[100];
@@ -1650,15 +1821,16 @@ TEST(GcsFileSystemTest, DeleteFile) {
TEST(GcsFileSystemTest, DeleteFile_NoObjectName) {
std::vector<HttpRequest*> requests;
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
fs.DeleteFile("gs://bucket/").code());
@@ -1696,11 +1868,12 @@ TEST(GcsFileSystemTest, DeleteFile_StatCacheRemoved) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 16 /* block size */, 16 /* max bytes */, 0 /* max staleness */,
- 3600 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 16 /* block size */,
+ 16 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
// Stats the file first so the stat is cached.
FileStatistics stat_before_deletion;
@@ -1721,15 +1894,16 @@ TEST(GcsFileSystemTest, DeleteDir_Empty) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"{}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/"));
}
@@ -1749,15 +1923,16 @@ TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) {
"Timeouts: 5 1 10\n"
"Delete: yes\n",
"")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/"));
}
@@ -1768,15 +1943,16 @@ TEST(GcsFileSystemTest, DeleteDir_BucketOnly) {
"name%2CnextPageToken&maxResults=2\nAuth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"{}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
TF_EXPECT_OK(fs.DeleteDir("gs://bucket"));
}
@@ -1789,15 +1965,16 @@ TEST(GcsFileSystemTest, DeleteDir_NonEmpty) {
"Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"path/file1.txt\" }]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
EXPECT_EQ(error::Code::FAILED_PRECONDITION,
fs.DeleteDir("gs://bucket/path/").code());
@@ -1811,15 +1988,16 @@ TEST(GcsFileSystemTest, GetFileSize) {
"Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
uint64 size;
TF_EXPECT_OK(fs.GetFileSize("gs://bucket/file.txt", &size));
@@ -1828,15 +2006,16 @@ TEST(GcsFileSystemTest, GetFileSize) {
TEST(GcsFileSystemTest, GetFileSize_NoObjectName) {
std::vector<HttpRequest*> requests;
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
uint64 size;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -1913,15 +2092,16 @@ TEST(GcsFileSystemTest, RenameFile_Folder) {
"Timeouts: 5 1 10\n"
"Delete: yes\n",
"")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
TF_EXPECT_OK(fs.RenameFile("gs://bucket/path1", "gs://bucket/path2/"));
}
@@ -2008,11 +2188,12 @@ TEST(GcsFileSystemTest, RenameFile_Object) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 16 /* block size */, 64 /* max bytes */, 0 /* max staleness */,
- 3600 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 16 /* block size */,
+ 64 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
// Do an initial read of the source and destination files to load their
// contents into the block cache.
char scratch[100];
@@ -2088,11 +2269,12 @@ TEST(GcsFileSystemTest, RenameFile_Object_FlushTargetStatCache) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 3600 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
// Do an initial stat of the destination file to load their contents into the
// stat cache.
FileStatistics stat_before_renaming;
@@ -2150,15 +2332,16 @@ TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) {
"Timeouts: 5 1 10\n"
"Delete: yes\n",
"", errors::NotFound("404"), 404)});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
TF_EXPECT_OK(
fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt"));
@@ -2191,15 +2374,16 @@ TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) {
"Post: yes\n"
"Timeouts: 5 1 10\n",
"{\"done\": false}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
EXPECT_EQ(
errors::Code::UNIMPLEMENTED,
@@ -2215,15 +2399,16 @@ TEST(GcsFileSystemTest, Stat_Object) {
"Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
FileStatistics stat;
TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat));
@@ -2248,15 +2433,16 @@ TEST(GcsFileSystemTest, Stat_Folder) {
"Timeouts: 5 1 10\n",
"{\"items\": [ "
" { \"name\": \"subfolder/\" }]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
FileStatistics stat;
TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", &stat));
@@ -2280,15 +2466,16 @@ TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"{}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
FileStatistics stat;
EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/path", &stat).code());
@@ -2300,15 +2487,16 @@ TEST(GcsFileSystemTest, Stat_Bucket) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"{}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
FileStatistics stat;
TF_EXPECT_OK(fs.Stat("gs://bucket/", &stat));
@@ -2323,15 +2511,16 @@ TEST(GcsFileSystemTest, Stat_BucketNotFound) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404)});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
FileStatistics stat;
EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/", &stat).code());
@@ -2364,11 +2553,12 @@ TEST(GcsFileSystemTest, Stat_Cache) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 3600 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
// Repeated calls to fs.Stat on these paths should not lead to any additional
// HTTP requests to GCS.
@@ -2405,11 +2595,12 @@ TEST(GcsFileSystemTest, Stat_Cache_Flush) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 3600 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 3600 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
- kTestTimeoutConfig, nullptr /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
// There should be a single HTTP request to GCS for fs.Stat in this loop.
for (int i = 0; i < 10; i++) {
FileStatistics stat;
@@ -2437,15 +2628,16 @@ TEST(GcsFileSystemTest, Stat_FilenameEndingWithSlash) {
"Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"5\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
FileStatistics stat;
TF_EXPECT_OK(fs.Stat("gs://bucket/dir/", &stat));
@@ -2468,15 +2660,16 @@ TEST(GcsFileSystemTest, IsDirectory_NotFound) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404)});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
EXPECT_EQ(error::Code::NOT_FOUND,
fs.IsDirectory("gs://bucket/file.txt").code());
@@ -2498,15 +2691,16 @@ TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) {
"Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
EXPECT_EQ(error::Code::FAILED_PRECONDITION,
fs.IsDirectory("gs://bucket/file.txt").code());
@@ -2528,15 +2722,16 @@ TEST(GcsFileSystemTest, IsDirectory_Yes) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"{\"items\": [{\"name\": \"subfolder/\"}]}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder"));
TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder/"));
@@ -2554,15 +2749,16 @@ TEST(GcsFileSystemTest, IsDirectory_Bucket) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"{}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
TF_EXPECT_OK(fs.IsDirectory("gs://bucket"));
TF_EXPECT_OK(fs.IsDirectory("gs://bucket/"));
@@ -2574,15 +2770,16 @@ TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404)});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/").code());
}
@@ -2615,15 +2812,16 @@ TEST(GcsFileSystemTest, CreateDir_Folder) {
"Timeouts: 5 1 30\n"
"Put body: \n",
"")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath"));
TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath/"));
@@ -2641,15 +2839,16 @@ TEST(GcsFileSystemTest, CreateDir_Bucket) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
TF_EXPECT_OK(fs.CreateDir("gs://bucket/"));
TF_EXPECT_OK(fs.CreateDir("gs://bucket"));
@@ -2712,15 +2911,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_Ok) {
"Timeouts: 5 1 10\n"
"Delete: yes\n",
"")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
int64 undeleted_files, undeleted_dirs;
TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files,
@@ -2804,15 +3004,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) {
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404)});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
int64 undeleted_files, undeleted_dirs;
TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files,
@@ -2838,15 +3039,16 @@ TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) {
"Auth Token: fake_token\n"
"Timeouts: 5 1 10\n",
"", errors::NotFound("404"), 404)});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay*/, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay*/,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
int64 undeleted_files, undeleted_dirs;
EXPECT_EQ(error::Code::NOT_FOUND,
@@ -2857,6 +3059,29 @@ TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) {
EXPECT_EQ(1, undeleted_dirs);
}
+TEST(GcsFileSystemTest, NoConstraintsEnvironmentVariableTest) {
+ unsetenv("GCS_ALLOWED_BUCKET_LOCATIONS");
+ // No constraints
+ GcsFileSystem fs1;
+ EXPECT_EQ(*kAllowedLocationsDefault, fs1.allowed_locations());
+
+ // Cover cache initialization code, any uninitialized cache will cause this to
+ // fail
+ fs1.FlushCaches();
+}
+
+TEST(GcsFileSystemTest, BucketLocationConstraintEnvironmentVariableTest) {
+ unsetenv("GCS_ALLOWED_BUCKET_LOCATIONS");
+ setenv("GCS_ALLOWED_BUCKET_LOCATIONS", "auto", 1);
+ GcsFileSystem fs1;
+ EXPECT_EQ(*kAllowedLocationsAuto, fs1.allowed_locations());
+
+ setenv("GCS_ALLOWED_BUCKET_LOCATIONS", "custom,list", 1);
+ GcsFileSystem fs2;
+ EXPECT_EQ(std::unordered_set<string>({"custom", "list"}),
+ fs2.allowed_locations());
+}
+
TEST(GcsFileSystemTest, AdditionalRequestHeaderTest) {
GcsFileSystem fs1;
EXPECT_EQ("", fs1.additional_header_name());
@@ -2902,11 +3127,12 @@ TEST(GcsFileSystemTest, AdditionalRequestHeaderTest) {
std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
0 /* matching paths cache max entries */, 0 /* initial retry delay */,
- kTestTimeoutConfig, add_header /* gcs additional header */);
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ add_header /* gcs additional header */);
std::unique_ptr<HttpRequest> request;
TF_EXPECT_OK(fs7.CreateHttpRequest(&request));
@@ -2973,15 +3199,16 @@ TEST(GcsFileSystemTest, CreateHttpRequest) {
"Auth Token: fake_token\n"
"Header Hello: world\n",
"{}")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
std::unique_ptr<HttpRequest> request;
TF_EXPECT_OK(fs.CreateHttpRequest(&request));
@@ -3035,15 +3262,16 @@ TEST(GcsFileSystemTest, Stat_StatsRecording) {
"Timeouts: 5 1 10\n",
strings::StrCat("{\"size\": \"1010\",\"generation\": \"1\","
"\"updated\": \"2016-04-29T23:15:24.896Z\"}"))});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
TestGcsStats stats;
fs.SetStats(&stats);
@@ -3061,15 +3289,16 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_StatsRecording) {
"Range: 0-5\n"
"Timeouts: 5 1 20\n",
"012345")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
- 0 /* stat cache max age */, 0 /* stat cache max entries */,
- 0 /* matching paths cache max age */,
- 0 /* matching paths cache max entries */,
- 0 /* initial retry delay */, kTestTimeoutConfig,
- nullptr /* gcs additional header */);
+ GcsFileSystem fs(
+ std::unique_ptr<AuthProvider>(new FakeAuthProvider),
+ std::unique_ptr<HttpRequest::Factory>(
+ new FakeHttpRequestFactory(&requests)),
+ std::unique_ptr<ZoneProvider>(new FakeZoneProvider), 0 /* block size */,
+ 0 /* max bytes */, 0 /* max staleness */, 0 /* stat cache max age */,
+ 0 /* stat cache max entries */, 0 /* matching paths cache max age */,
+ 0 /* matching paths cache max entries */, 0 /* initial retry delay */,
+ kTestTimeoutConfig, *kAllowedLocationsDefault,
+ nullptr /* gcs additional header */);
TestGcsStats stats;
fs.SetStats(&stats);