aboutsummaryrefslogtreecommitdiffhomepage
path: root/tensorflow/core/platform/cloud/gcs_file_system_test.cc
diff options
context:
space:
mode:
Diffstat (limited to 'tensorflow/core/platform/cloud/gcs_file_system_test.cc')
-rw-r--r--tensorflow/core/platform/cloud/gcs_file_system_test.cc500
1 files changed, 281 insertions, 219 deletions
diff --git a/tensorflow/core/platform/cloud/gcs_file_system_test.cc b/tensorflow/core/platform/cloud/gcs_file_system_test.cc
index 915a44ad7f..ba08c7414f 100644
--- a/tensorflow/core/platform/cloud/gcs_file_system_test.cc
+++ b/tensorflow/core/platform/cloud/gcs_file_system_test.cc
@@ -45,8 +45,8 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::unique_ptr<RandomAccessFile> file;
TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
@@ -80,8 +80,8 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_differentN) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::unique_ptr<RandomAccessFile> file;
TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file));
@@ -130,7 +130,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 9 /* block size */, 2 /* block count */,
+ 9 /* block size */, 18 /* max bytes */,
0 /* max staleness */, 0 /* initial retry delay */);
char scratch[100];
@@ -196,7 +196,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_MaxStaleness) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 8 /* block size */, 2 /* block count */,
+ 8 /* block size */, 16 /* max bytes */,
3600 /* max staleness */, 0 /* initial retry delay */);
char scratch[100];
StringPiece result;
@@ -205,7 +205,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_MaxStaleness) {
// across file close/open boundaries.
for (int i = 0; i < 10; i++) {
// Create two files. Since these files have the same name name and the max
- // staleness of the filesystem is > 0, they will share the same block cache.
+ // staleness of the filesystem is > 0, they will share the same blocks.
std::unique_ptr<RandomAccessFile> file1;
std::unique_ptr<RandomAccessFile> file2;
TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/object", &file1));
@@ -234,7 +234,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoObjectName) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* read ahead bytes */, 0 /* block count */,
+ 0 /* read ahead bytes */, 0 /* max bytes */,
0 /* max staleness */, 0 /* initial retry delay */);
std::unique_ptr<RandomAccessFile> file;
@@ -242,104 +242,57 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoObjectName) {
fs.NewRandomAccessFile("gs://bucket/", &file).code());
}
-TEST(GcsFileSystemTest, NewWritableFile_FlushesFileCache) {
- // Our underlying file in this test is a 16 byte file with initial contents
- // "0123456789abcdef", replaced by new contents "08192a3b4c5d6e7f".
+TEST(GcsFileSystemTest, NewWritableFile) {
std::vector<HttpRequest*> requests(
- {new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n"
- "Auth Token: fake_token\n"
- "Range: 0-7\n",
- "01234567"),
- new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n"
- "Auth Token: fake_token\n"
- "Range: 8-15\n",
- "89abcdef"),
+ {new FakeHttpRequest(
+ "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
+ "Auth Token: fake_token\n"
+ "Range: 0-7\n",
+ "01234567"),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
- "uploadType=resumable&name=object\n"
+ "uploadType=resumable&name=path%2Fwriteable\n"
"Auth Token: fake_token\n"
- "Header X-Upload-Content-Length: 16\n"
+ "Header X-Upload-Content-Length: 17\n"
"Post: yes\n",
"", {{"Location", "https://custom/upload/location"}}),
new FakeHttpRequest("Uri: https://custom/upload/location\n"
"Auth Token: fake_token\n"
- "Header Content-Range: bytes 0-15/16\n"
- "Put body: 08192a3b4c5d6e7f\n",
+ "Header Content-Range: bytes 0-16/17\n"
+ "Put body: content1,content2\n",
""),
- new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n"
- "Auth Token: fake_token\n"
- "Range: 0-7\n",
- "08192a3b"),
- new FakeHttpRequest("Uri: https://storage.googleapis.com/bucket/object\n"
- "Auth Token: fake_token\n"
- "Range: 8-15\n",
- "4c5d6e7f")});
+ new FakeHttpRequest(
+ "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
+ "Auth Token: fake_token\n"
+ "Range: 0-7\n",
+ "01234567")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 8 /* block size */, 2 /* block count */,
- 3600 /* max staleness */, 0 /* initial retry delay */);
- // First, read both blocks of the file with its initial contents. This should
- // trigger the first two HTTP requests to GCS, to load each of the blocks in
- // order.
+ 8 /* block size */, 8 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
+
+ // Read from the file first, to fill the block cache.
std::unique_ptr<RandomAccessFile> rfile;
- TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/object", &rfile));
+ TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/writeable", &rfile));
char scratch[100];
StringPiece result;
- TF_EXPECT_OK(rfile->Read(0, 16, &result, scratch));
- EXPECT_EQ("0123456789abcdef", result);
- // Now open a writable file and write the new file contents. This should
- // trigger the next two HTTP requests to GCS.
+ TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
+ EXPECT_EQ("0123", result);
+ // Open the writable file.
std::unique_ptr<WritableFile> wfile;
- TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/object", &wfile));
- TF_EXPECT_OK(wfile->Append("08192a3b4c5d6e7f"));
- // This rfile read should hit the block cache and not trigger any requests,
- // because we haven't flushed the write yet.
- TF_EXPECT_OK(rfile->Read(0, 16, &result, scratch));
- EXPECT_EQ("0123456789abcdef", result);
- // Now flush the write. Any opens of gs://bucket/object after this point will
- // no longer use the previous FileBlockCache for that file.
+ TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable", &wfile));
+ TF_EXPECT_OK(wfile->Append("content1,"));
+ TF_EXPECT_OK(wfile->Append("content2"));
TF_EXPECT_OK(wfile->Flush());
- // Reopen `rfile` and read its contents again. This will trigger the next two
- // HTTP requests to GCS, which will return the (updated) file contents. Note
- // that the contents returned in those HTTP requests are not really relevant;
- // the main thing is that both blocks of the file are reloaded.
- TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/object", &rfile));
- TF_EXPECT_OK(rfile->Read(0, 16, &result, scratch));
- EXPECT_EQ("08192a3b4c5d6e7f", result);
-}
-
-TEST(GcsFileSystemTest, NewWritableFile) {
- std::vector<HttpRequest*> requests(
- {new FakeHttpRequest(
- "Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
- "uploadType=resumable&name=path%2Fwriteable.txt\n"
- "Auth Token: fake_token\n"
- "Header X-Upload-Content-Length: 17\n"
- "Post: yes\n",
- "", {{"Location", "https://custom/upload/location"}}),
- new FakeHttpRequest("Uri: https://custom/upload/location\n"
- "Auth Token: fake_token\n"
- "Header Content-Range: bytes 0-16/17\n"
- "Put body: content1,content2\n",
- "")});
- GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
- std::unique_ptr<HttpRequest::Factory>(
- new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
-
- std::unique_ptr<WritableFile> file;
- TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file));
-
- TF_EXPECT_OK(file->Append("content1,"));
- TF_EXPECT_OK(file->Append("content2"));
- TF_EXPECT_OK(file->Flush());
+ // Re-reading the file should trigger another HTTP request to GCS.
+ TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
+ EXPECT_EQ("0123", result);
// The calls to flush, sync, and close below should not cause uploads because
// the file is not dirty.
- TF_EXPECT_OK(file->Flush());
- TF_EXPECT_OK(file->Sync());
- TF_EXPECT_OK(file->Close());
+ TF_EXPECT_OK(wfile->Flush());
+ TF_EXPECT_OK(wfile->Sync());
+ TF_EXPECT_OK(wfile->Close());
}
TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) {
@@ -381,8 +334,8 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::unique_ptr<WritableFile> file;
TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file));
@@ -393,10 +346,18 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) {
}
TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceedsOnGetStatus) {
+ // This test also verifies that a file's blocks are purged from the cache when
+ // the file is written, even when the write takes the "succeeds on get status"
+ // path.
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
+ "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
+ "Auth Token: fake_token\n"
+ "Range: 0-7\n",
+ "01234567"),
+ new FakeHttpRequest(
"Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
- "uploadType=resumable&name=path%2Fwriteable.txt\n"
+ "uploadType=resumable&name=path%2Fwriteable\n"
"Auth Token: fake_token\n"
"Header X-Upload-Content-Length: 17\n"
"Post: yes\n",
@@ -410,19 +371,41 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceedsOnGetStatus) {
"Auth Token: fake_token\n"
"Header Content-Range: bytes */17\n"
"Put: yes\n",
- "", Status::OK(), nullptr, {}, 201)});
+ "", Status::OK(), nullptr, {}, 201),
+ new FakeHttpRequest(
+ "Uri: https://storage.googleapis.com/bucket/path%2Fwriteable\n"
+ "Auth Token: fake_token\n"
+ "Range: 0-7\n",
+ "01234567")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
-
- std::unique_ptr<WritableFile> file;
- TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file));
-
- TF_EXPECT_OK(file->Append("content1,"));
- TF_EXPECT_OK(file->Append("content2"));
- TF_EXPECT_OK(file->Close());
+ 8 /* block size */, 8 /* max bytes */,
+ 3600 /* max staleness */, 0 /* initial retry delay */);
+ // Pull the file's first block into the cache. This will trigger the first
+ // HTTP request to GCS.
+ std::unique_ptr<RandomAccessFile> rfile;
+ TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/writeable", &rfile));
+ char scratch[100];
+ StringPiece result;
+ TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
+ EXPECT_EQ("0123", result);
+ // Now write to the same file. Once the write succeeds, the cached block will
+ // be flushed.
+ std::unique_ptr<WritableFile> wfile;
+ TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable", &wfile));
+ TF_EXPECT_OK(wfile->Append("content1,"));
+ TF_EXPECT_OK(wfile->Append("content2"));
+ // Appending doesn't invalidate the read cache - only flushing does. This read
+ // will not trigger an HTTP request to GCS.
+ TF_EXPECT_OK(rfile->Read(4, 4, &result, scratch));
+ EXPECT_EQ("4567", result);
+ // Closing the file triggers HTTP requests to GCS and invalidates the read
+ // cache for the file.
+ TF_EXPECT_OK(wfile->Close());
+ // Reading the first block of the file goes to GCS again.
+ TF_EXPECT_OK(rfile->Read(0, 8, &result, scratch));
+ EXPECT_EQ("01234567", result);
}
TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) {
@@ -472,8 +455,8 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 2 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 2 /* initial retry delay */);
std::unique_ptr<WritableFile> file;
TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file));
@@ -520,8 +503,8 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::unique_ptr<WritableFile> file;
TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file));
@@ -546,8 +529,8 @@ TEST(GcsFileSystemTest, NewWritableFile_NoObjectName) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::unique_ptr<WritableFile> file;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -557,13 +540,13 @@ TEST(GcsFileSystemTest, NewWritableFile_NoObjectName) {
TEST(GcsFileSystemTest, NewAppendableFile) {
std::vector<HttpRequest*> requests(
{new FakeHttpRequest(
- "Uri: https://storage.googleapis.com/bucket/path%2Fappendable.txt\n"
+ "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
"Auth Token: fake_token\n"
- "Range: 0-1048575\n",
+ "Range: 0-31\n",
"content1,"),
new FakeHttpRequest(
"Uri: https://www.googleapis.com/upload/storage/v1/b/bucket/o?"
- "uploadType=resumable&name=path%2Fappendable.txt\n"
+ "uploadType=resumable&name=path%2Fappendable\n"
"Auth Token: fake_token\n"
"Header X-Upload-Content-Length: 17\n"
"Post: yes\n",
@@ -572,18 +555,38 @@ TEST(GcsFileSystemTest, NewAppendableFile) {
"Auth Token: fake_token\n"
"Header Content-Range: bytes 0-16/17\n"
"Put body: content1,content2\n",
- "")});
+ ""),
+ new FakeHttpRequest(
+ "Uri: https://storage.googleapis.com/bucket/path%2Fappendable\n"
+ "Auth Token: fake_token\n"
+ "Range: 0-31\n",
+ "01234567")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
+ 32 /* block size */, 32 /* max bytes */,
0 /* max staleness */, 0 /* initial retry delay */);
- std::unique_ptr<WritableFile> file;
- TF_EXPECT_OK(fs.NewAppendableFile("gs://bucket/path/appendable.txt", &file));
-
- TF_EXPECT_OK(file->Append("content2"));
- TF_EXPECT_OK(file->Close());
+ // Create an appendable file. This should read the file from GCS, and pull its
+ // contents into the block cache.
+ std::unique_ptr<WritableFile> wfile;
+ TF_EXPECT_OK(fs.NewAppendableFile("gs://bucket/path/appendable", &wfile));
+ TF_EXPECT_OK(wfile->Append("content2"));
+ // Verify that the file contents are in the block cache. This read should not
+ // trigger an HTTP request to GCS.
+ std::unique_ptr<RandomAccessFile> rfile;
+ TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/appendable", &rfile));
+ char scratch[100];
+ StringPiece result;
+ TF_EXPECT_OK(rfile->Read(0, 8, &result, scratch));
+ EXPECT_EQ("content1", result);
+ // Closing the appendable file will flush its contents to GCS, triggering HTTP
+ // requests.
+ TF_EXPECT_OK(wfile->Close());
+ // Redo the read. The block should be reloaded from GCS, causing one more HTTP
+ // request to load it.
+ TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch));
+ EXPECT_EQ("0123", result);
}
TEST(GcsFileSystemTest, NewAppendableFile_NoObjectName) {
@@ -591,8 +594,8 @@ TEST(GcsFileSystemTest, NewAppendableFile_NoObjectName) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::unique_ptr<WritableFile> file;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -618,8 +621,8 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::unique_ptr<ReadOnlyMemoryRegion> region;
TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile(
@@ -634,8 +637,8 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile_NoObjectName) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::unique_ptr<ReadOnlyMemoryRegion> region;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -652,8 +655,8 @@ TEST(GcsFileSystemTest, FileExists_YesAsObject) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt"));
}
@@ -675,8 +678,8 @@ TEST(GcsFileSystemTest, FileExists_YesAsFolder) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder"));
}
@@ -694,8 +697,8 @@ TEST(GcsFileSystemTest, FileExists_YesAsBucket) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
TF_EXPECT_OK(fs.FileExists("gs://bucket1"));
TF_EXPECT_OK(fs.FileExists("gs://bucket1/"));
@@ -717,8 +720,8 @@ TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
EXPECT_EQ(errors::Code::NOT_FOUND,
fs.FileExists("gs://bucket/path/file1.txt").code());
@@ -737,8 +740,8 @@ TEST(GcsFileSystemTest, FileExists_NotAsBucket) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
fs.FileExists("gs://bucket2/").code());
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -755,8 +758,8 @@ TEST(GcsFileSystemTest, GetChildren_NoItems) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
@@ -777,8 +780,8 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
@@ -800,8 +803,8 @@ TEST(GcsFileSystemTest, GetChildren_SelfDirectoryMarker) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
@@ -822,8 +825,8 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles_NoSlash) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children));
@@ -841,8 +844,8 @@ TEST(GcsFileSystemTest, GetChildren_Root) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket-a-b-c", &children));
@@ -860,8 +863,8 @@ TEST(GcsFileSystemTest, GetChildren_Empty) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children));
@@ -894,8 +897,8 @@ TEST(GcsFileSystemTest, GetChildren_Pagination) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::vector<string> children;
TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children));
@@ -915,8 +918,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_NoWildcard) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::vector<string> result;
TF_EXPECT_OK(
@@ -937,8 +940,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_BucketAndWildcard) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::vector<string> result;
TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", &result));
@@ -960,8 +963,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_Matches) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::vector<string> result;
TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file2.txt", &result));
@@ -980,8 +983,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_SelfDirectoryMarker) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::vector<string> result;
TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", &result));
@@ -1000,8 +1003,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::vector<string> result;
TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file3.txt", &result));
@@ -1013,8 +1016,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_OnlyWildcard) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
std::vector<string> result;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -1023,18 +1026,40 @@ TEST(GcsFileSystemTest, GetMatchingPaths_OnlyWildcard) {
TEST(GcsFileSystemTest, DeleteFile) {
std::vector<HttpRequest*> requests(
- {new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
+ {new FakeHttpRequest(
+ "Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n"
+ "Auth Token: fake_token\n"
+ "Range: 0-15\n",
+ "01234567"),
+ new FakeHttpRequest("Uri: https://www.googleapis.com/storage/v1/b"
"/bucket/o/path%2Ffile1.txt\n"
"Auth Token: fake_token\n"
"Delete: yes\n",
- "")});
+ ""),
+ new FakeHttpRequest(
+ "Uri: https://storage.googleapis.com/bucket/path%2Ffile1.txt\n"
+ "Auth Token: fake_token\n"
+ "Range: 0-15\n",
+ "76543210")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
+ 16 /* block size */, 16 /* max bytes */,
0 /* max staleness */, 0 /* initial retry delay */);
+ // Do an initial read of the file to load its contents into the block cache.
+ char scratch[100];
+ StringPiece result;
+ std::unique_ptr<RandomAccessFile> file;
+ TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/file1.txt", &file));
+ TF_EXPECT_OK(file->Read(0, 8, &result, scratch));
+ EXPECT_EQ("01234567", result);
+ // Deleting the file triggers the next HTTP request to GCS.
TF_EXPECT_OK(fs.DeleteFile("gs://bucket/path/file1.txt"));
+ // Re-reading the file causes its contents to be reloaded from GCS and not
+ // from the block cache.
+ TF_EXPECT_OK(file->Read(0, 8, &result, scratch));
+ EXPECT_EQ("76543210", result);
}
TEST(GcsFileSystemTest, DeleteFile_NoObjectName) {
@@ -1042,8 +1067,8 @@ TEST(GcsFileSystemTest, DeleteFile_NoObjectName) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
fs.DeleteFile("gs://bucket/").code());
@@ -1058,8 +1083,8 @@ TEST(GcsFileSystemTest, DeleteDir_Empty) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/"));
}
@@ -1080,8 +1105,8 @@ TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/"));
}
@@ -1094,8 +1119,8 @@ TEST(GcsFileSystemTest, DeleteDir_BucketOnly) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
TF_EXPECT_OK(fs.DeleteDir("gs://bucket"));
}
@@ -1110,8 +1135,8 @@ TEST(GcsFileSystemTest, DeleteDir_NonEmpty) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
EXPECT_EQ(error::Code::FAILED_PRECONDITION,
fs.DeleteDir("gs://bucket/path/").code());
@@ -1127,8 +1152,8 @@ TEST(GcsFileSystemTest, GetFileSize) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
uint64 size;
TF_EXPECT_OK(fs.GetFileSize("gs://bucket/file.txt", &size));
@@ -1140,8 +1165,8 @@ TEST(GcsFileSystemTest, GetFileSize_NoObjectName) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
uint64 size;
EXPECT_EQ(errors::Code::INVALID_ARGUMENT,
@@ -1213,15 +1238,25 @@ TEST(GcsFileSystemTest, RenameFile_Folder) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
TF_EXPECT_OK(fs.RenameFile("gs://bucket/path1", "gs://bucket/path2/"));
}
TEST(GcsFileSystemTest, RenameFile_Object) {
std::vector<HttpRequest*> requests(
- {// IsDirectory is checking whether there are children objects.
+ {new FakeHttpRequest(
+ "Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n"
+ "Auth Token: fake_token\n"
+ "Range: 0-15\n",
+ "01234567"),
+ new FakeHttpRequest(
+ "Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n"
+ "Auth Token: fake_token\n"
+ "Range: 0-15\n",
+ "76543210"),
+ // IsDirectory is checking whether there are children objects.
new FakeHttpRequest(
"Uri: https://www.googleapis.com/storage/v1/b/bucket/o?"
"fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F"
@@ -1248,15 +1283,42 @@ TEST(GcsFileSystemTest, RenameFile_Object) {
"path%2Fsrc.txt\n"
"Auth Token: fake_token\n"
"Delete: yes\n",
- "")});
+ ""),
+ new FakeHttpRequest(
+ "Uri: https://storage.googleapis.com/bucket/path%2Fsrc.txt\n"
+ "Auth Token: fake_token\n"
+ "Range: 0-15\n",
+ "89abcdef"),
+ new FakeHttpRequest(
+ "Uri: https://storage.googleapis.com/bucket/path%2Fdst.txt\n"
+ "Auth Token: fake_token\n"
+ "Range: 0-15\n",
+ "fedcba98")});
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
+ 16 /* block size */, 64 /* max bytes */,
0 /* max staleness */, 0 /* initial retry delay */);
-
+ // Do an initial read of the source and destination files to load their
+ // contents into the block cache.
+ char scratch[100];
+ StringPiece result;
+ std::unique_ptr<RandomAccessFile> src;
+ std::unique_ptr<RandomAccessFile> dst;
+ TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/src.txt", &src));
+ TF_EXPECT_OK(src->Read(0, 8, &result, scratch));
+ EXPECT_EQ("01234567", result);
+ TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/dst.txt", &dst));
+ TF_EXPECT_OK(dst->Read(0, 8, &result, scratch));
+ EXPECT_EQ("76543210", result);
+ // Now rename src to dst. This should flush the block cache for both files.
TF_EXPECT_OK(
fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt"));
+ // Re-read both files. This should reload their contents from GCS.
+ TF_EXPECT_OK(src->Read(0, 8, &result, scratch));
+ EXPECT_EQ("89abcdef", result);
+ TF_EXPECT_OK(dst->Read(0, 8, &result, scratch));
+ EXPECT_EQ("fedcba98", result);
}
/// Tests the scenario when deletion returns a failure, but actually succeeds.
@@ -1300,8 +1362,8 @@ TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
TF_EXPECT_OK(
fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt"));
@@ -1334,8 +1396,8 @@ TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
EXPECT_EQ(
errors::Code::UNIMPLEMENTED,
@@ -1353,8 +1415,8 @@ TEST(GcsFileSystemTest, Stat_Object) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
FileStatistics stat;
TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat));
@@ -1380,8 +1442,8 @@ TEST(GcsFileSystemTest, Stat_Folder) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
FileStatistics stat;
TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", &stat));
@@ -1406,8 +1468,8 @@ TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
FileStatistics stat;
EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/path", &stat).code());
@@ -1421,8 +1483,8 @@ TEST(GcsFileSystemTest, Stat_Bucket) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
FileStatistics stat;
TF_EXPECT_OK(fs.Stat("gs://bucket/", &stat));
@@ -1439,8 +1501,8 @@ TEST(GcsFileSystemTest, Stat_BucketNotFound) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
FileStatistics stat;
EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/", &stat).code());
@@ -1462,8 +1524,8 @@ TEST(GcsFileSystemTest, IsDirectory_NotFound) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
EXPECT_EQ(error::Code::NOT_FOUND,
fs.IsDirectory("gs://bucket/file.txt").code());
@@ -1486,8 +1548,8 @@ TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
EXPECT_EQ(error::Code::FAILED_PRECONDITION,
fs.IsDirectory("gs://bucket/file.txt").code());
@@ -1510,8 +1572,8 @@ TEST(GcsFileSystemTest, IsDirectory_Yes) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder"));
TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder/"));
@@ -1530,8 +1592,8 @@ TEST(GcsFileSystemTest, IsDirectory_Bucket) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
TF_EXPECT_OK(fs.IsDirectory("gs://bucket"));
TF_EXPECT_OK(fs.IsDirectory("gs://bucket/"));
@@ -1545,8 +1607,8 @@ TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/").code());
}
@@ -1578,8 +1640,8 @@ TEST(GcsFileSystemTest, CreateDir_Folder) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath"));
TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath/"));
@@ -1598,8 +1660,8 @@ TEST(GcsFileSystemTest, CreateDir_Bucket) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
TF_EXPECT_OK(fs.CreateDir("gs://bucket/"));
TF_EXPECT_OK(fs.CreateDir("gs://bucket"));
@@ -1658,8 +1720,8 @@ TEST(GcsFileSystemTest, DeleteRecursively_Ok) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
int64 undeleted_files, undeleted_dirs;
TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files,
@@ -1737,8 +1799,8 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
int64 undeleted_files, undeleted_dirs;
TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files,
@@ -1765,8 +1827,8 @@ TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) {
GcsFileSystem fs(std::unique_ptr<AuthProvider>(new FakeAuthProvider),
std::unique_ptr<HttpRequest::Factory>(
new FakeHttpRequestFactory(&requests)),
- 0 /* block size */, 0 /* block count */,
- 0 /* max staleness */, 0 /* initial retry delay */);
+ 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */,
+ 0 /* initial retry delay */);
int64 undeleted_files, undeleted_dirs;
EXPECT_EQ(error::Code::NOT_FOUND,
@@ -1781,7 +1843,7 @@ TEST(GcsFileSystemTest, OverrideCacheParameters) {
// Verify defaults are propagated correctly.
GcsFileSystem fs1;
EXPECT_EQ(256 * 1024 * 1024, fs1.block_size());
- EXPECT_EQ(1, fs1.block_count());
+ EXPECT_EQ(fs1.block_size(), fs1.max_bytes());
EXPECT_EQ(0, fs1.max_staleness());
// Verify legacy readahead buffer override sets block size.
@@ -1794,10 +1856,10 @@ TEST(GcsFileSystemTest, OverrideCacheParameters) {
GcsFileSystem fs3;
EXPECT_EQ(1048576L, fs3.block_size());
- // Verify block count override.
- setenv("GCS_READ_CACHE_BLOCK_COUNT", "16", 1);
+ // Verify max size override.
+ setenv("GCS_READ_CACHE_MAX_SIZE_MB", "16", 1);
GcsFileSystem fs4;
- EXPECT_EQ(16, fs4.block_count());
+ EXPECT_EQ(16 * 1024 * 1024, fs4.max_bytes());
// Verify max staleness override.
setenv("GCS_READ_CACHE_MAX_STALENESS", "60", 1);
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-07 09:40:27 +0000