diff options
author | Ruoxin Sang <rxsang@google.com> | 2018-05-03 17:21:26 -0700 |
---|---|---|
committer | TensorFlower Gardener <gardener@tensorflow.org> | 2018-05-04 10:38:48 -0700 |
commit | fc7b593cda65f4a3a3de0cc733270f0864f820e2 (patch) | |
tree | d27112c5fc2ece5f84abd098079e3bd67f17e2a2 /tensorflow/core/platform | |
parent | 213a98d893105945540e0169faa124ac7e1200ba (diff) |
Clear the stat cache of the target when renaming the file.
PiperOrigin-RevId: 195337886
Diffstat (limited to 'tensorflow/core/platform')
-rw-r--r-- | tensorflow/core/platform/cloud/gcs_file_system.cc | 4 | ||||
-rw-r--r-- | tensorflow/core/platform/cloud/gcs_file_system_test.cc | 72 |
2 files changed, 74 insertions, 2 deletions
diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc index 488f9cc75d..e44e897434 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system.cc @@ -1375,9 +1375,9 @@ Status GcsFileSystem::RenameObject(const string& src, const string& target) { request->SetResultBuffer(&output_buffer); TF_RETURN_WITH_CONTEXT_IF_ERROR(request->Send(), " when renaming ", src, " to ", target); - // Flush the target from the block cache. The source will be flushed in the + // Flush the target from the caches. The source will be flushed in the // DeleteFile call below. - file_block_cache_->RemoveFile(target); + ClearFileCaches(target); Json::Value root; TF_RETURN_IF_ERROR(ParseJson(output_buffer, &root)); bool done; diff --git a/tensorflow/core/platform/cloud/gcs_file_system_test.cc b/tensorflow/core/platform/cloud/gcs_file_system_test.cc index c639299954..28be13869b 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system_test.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system_test.cc @@ -1902,6 +1902,78 @@ TEST(GcsFileSystemTest, RenameFile_Object) { EXPECT_EQ("fedcba98", result); } +TEST(GcsFileSystemTest, RenameFile_Object_FlushTargetStatCache) { + std::vector<HttpRequest*> requests( + {// Stat the target file. + new FakeHttpRequest( + "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" + "path%2Fdst.txt?fields=size%2Cupdated\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", + strings::StrCat("{\"size\": \"1000\"," + "\"updated\": \"2016-04-29T23:15:24.896Z\"}")), + // IsDirectory is checking whether there are children objects. + new FakeHttpRequest( + "Uri: https://www.googleapis.com/storage/v1/b/bucket/o?" + "fields=items%2Fname%2CnextPageToken&prefix=path%2Fsrc.txt%2F" + "&maxResults=1\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", + "{}"), + // IsDirectory is checking if the path exists as an object. + new FakeHttpRequest( + "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" + "path%2Fsrc.txt?fields=size%2Cupdated\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", + strings::StrCat("{\"size\": \"1010\"," + "\"updated\": \"2016-04-29T23:15:24.896Z\"}")), + // Copying to the new location. + new FakeHttpRequest( + "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" + "path%2Fsrc.txt/rewriteTo/b/bucket/o/path%2Fdst.txt\n" + "Auth Token: fake_token\n" + "Post: yes\n" + "Timeouts: 5 1 10\n", + "{\"done\": true}"), + // Deleting the original file. + new FakeHttpRequest( + "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" + "path%2Fsrc.txt\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n" + "Delete: yes\n", + ""), + new FakeHttpRequest( + "Uri: https://www.googleapis.com/storage/v1/b/bucket/o/" + "path%2Fdst.txt?fields=size%2Cupdated\n" + "Auth Token: fake_token\n" + "Timeouts: 5 1 10\n", + strings::StrCat("{\"size\": \"1010\"," + "\"updated\": \"2016-04-29T23:15:24.896Z\"}"))}); + GcsFileSystem fs( + std::unique_ptr<AuthProvider>(new FakeAuthProvider), + std::unique_ptr<HttpRequest::Factory>( + new FakeHttpRequestFactory(&requests)), + 0 /* block size */, 0 /* max bytes */, 0 /* max staleness */, + 3600 /* stat cache max age */, 0 /* stat cache max entries */, + 0 /* matching paths cache max age */, + 0 /* matching paths cache max entries */, 0 /* initial retry delay*/, + kTestTimeoutConfig, nullptr /* gcs additional header */); + // Do an initial stat of the destination file to load their contents into the + // stat cache. + FileStatistics stat_before_renaming; + TF_EXPECT_OK(fs.Stat("gs://bucket/path/dst.txt", &stat_before_renaming)); + EXPECT_EQ(1000, stat_before_renaming.length); + + TF_EXPECT_OK( + fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt")); + + FileStatistics stat_after_renaming; + TF_EXPECT_OK(fs.Stat("gs://bucket/path/dst.txt", &stat_after_renaming)); + EXPECT_EQ(1010, stat_after_renaming.length); +} + /// Tests the scenario when deletion returns a failure, but actually succeeds. TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) { std::vector<HttpRequest*> requests( |