diff options
Diffstat (limited to 'tensorflow/core/platform/hadoop/hadoop_file_system.cc')
-rw-r--r-- | tensorflow/core/platform/hadoop/hadoop_file_system.cc | 17 |
1 files changed, 15 insertions, 2 deletions
diff --git a/tensorflow/core/platform/hadoop/hadoop_file_system.cc b/tensorflow/core/platform/hadoop/hadoop_file_system.cc index 7ae63ac1d2..745eb9e6cd 100644 --- a/tensorflow/core/platform/hadoop/hadoop_file_system.cc +++ b/tensorflow/core/platform/hadoop/hadoop_file_system.cc @@ -56,6 +56,8 @@ class LibHDFS { std::function<hdfsFS(hdfsBuilder*)> hdfsBuilderConnect; std::function<hdfsBuilder*()> hdfsNewBuilder; std::function<void(hdfsBuilder*, const char*)> hdfsBuilderSetNameNode; + std::function<void(hdfsBuilder*, const char *kerbTicketCachePath)> + hdfsBuilderSetKerbTicketCachePath; std::function<int(hdfsFS, hdfsFile)> hdfsCloseFile; std::function<tSize(hdfsFS, hdfsFile, tOffset, void*, tSize)> hdfsPread; std::function<tSize(hdfsFS, hdfsFile, const void*, tSize)> hdfsWrite; @@ -81,6 +83,7 @@ class LibHDFS { BIND_HDFS_FUNC(hdfsBuilderConnect); BIND_HDFS_FUNC(hdfsNewBuilder); BIND_HDFS_FUNC(hdfsBuilderSetNameNode); + BIND_HDFS_FUNC(hdfsBuilderSetKerbTicketCachePath); BIND_HDFS_FUNC(hdfsCloseFile); BIND_HDFS_FUNC(hdfsPread); BIND_HDFS_FUNC(hdfsWrite); @@ -135,6 +138,10 @@ Status HadoopFileSystem::Connect(StringPiece fname, hdfsFS* fs) { } else { hdfs_->hdfsBuilderSetNameNode(builder, nn.c_str()); } + char* ticket_cache_path = getenv("KERB_TICKET_CACHE_PATH"); + if (ticket_cache_path != nullptr) { + hdfs_->hdfsBuilderSetKerbTicketCachePath(builder, ticket_cache_path); + } *fs = hdfs_->hdfsBuilderConnect(builder); if (*fs == nullptr) { return errors::NotFound(strerror(errno)); @@ -360,9 +367,15 @@ Status HadoopFileSystem::DeleteDir(const string& dir) { hdfsFileInfo* info = hdfs_->hdfsListDirectory(fs, TranslateName(dir).c_str(), &entries); if (info != nullptr) { - return IOError(dir, errno); + hdfs_->hdfsFreeFileInfo(info, entries); + } + // Due to HDFS bug HDFS-8407, we can't distinguish between an error and empty + // folder, expscially for Kerberos enable setup, EAGAIN is quite common when + // the call is actually successful. Check again by Stat. + if (info == nullptr && errno != 0) { + FileStatistics stat; + TF_RETURN_IF_ERROR(Stat(dir, &stat)); } - hdfs_->hdfsFreeFileInfo(info, entries); if (entries > 0) { return errors::FailedPrecondition("Cannot delete a non-empty directory."); |