aboutsummaryrefslogtreecommitdiffhomepage
path: root/tensorflow/core/platform/hadoop/hadoop_file_system.cc
diff options
context:
space:
mode:
Diffstat (limited to 'tensorflow/core/platform/hadoop/hadoop_file_system.cc')
-rw-r--r--tensorflow/core/platform/hadoop/hadoop_file_system.cc17
1 files changed, 15 insertions, 2 deletions
diff --git a/tensorflow/core/platform/hadoop/hadoop_file_system.cc b/tensorflow/core/platform/hadoop/hadoop_file_system.cc
index 7ae63ac1d2..745eb9e6cd 100644
--- a/tensorflow/core/platform/hadoop/hadoop_file_system.cc
+++ b/tensorflow/core/platform/hadoop/hadoop_file_system.cc
@@ -56,6 +56,8 @@ class LibHDFS {
std::function<hdfsFS(hdfsBuilder*)> hdfsBuilderConnect;
std::function<hdfsBuilder*()> hdfsNewBuilder;
std::function<void(hdfsBuilder*, const char*)> hdfsBuilderSetNameNode;
+ std::function<void(hdfsBuilder*, const char *kerbTicketCachePath)>
+ hdfsBuilderSetKerbTicketCachePath;
std::function<int(hdfsFS, hdfsFile)> hdfsCloseFile;
std::function<tSize(hdfsFS, hdfsFile, tOffset, void*, tSize)> hdfsPread;
std::function<tSize(hdfsFS, hdfsFile, const void*, tSize)> hdfsWrite;
@@ -81,6 +83,7 @@ class LibHDFS {
BIND_HDFS_FUNC(hdfsBuilderConnect);
BIND_HDFS_FUNC(hdfsNewBuilder);
BIND_HDFS_FUNC(hdfsBuilderSetNameNode);
+ BIND_HDFS_FUNC(hdfsBuilderSetKerbTicketCachePath);
BIND_HDFS_FUNC(hdfsCloseFile);
BIND_HDFS_FUNC(hdfsPread);
BIND_HDFS_FUNC(hdfsWrite);
@@ -135,6 +138,10 @@ Status HadoopFileSystem::Connect(StringPiece fname, hdfsFS* fs) {
} else {
hdfs_->hdfsBuilderSetNameNode(builder, nn.c_str());
}
+ char* ticket_cache_path = getenv("KERB_TICKET_CACHE_PATH");
+ if (ticket_cache_path != nullptr) {
+ hdfs_->hdfsBuilderSetKerbTicketCachePath(builder, ticket_cache_path);
+ }
*fs = hdfs_->hdfsBuilderConnect(builder);
if (*fs == nullptr) {
return errors::NotFound(strerror(errno));
@@ -360,9 +367,15 @@ Status HadoopFileSystem::DeleteDir(const string& dir) {
hdfsFileInfo* info =
hdfs_->hdfsListDirectory(fs, TranslateName(dir).c_str(), &entries);
if (info != nullptr) {
- return IOError(dir, errno);
+ hdfs_->hdfsFreeFileInfo(info, entries);
+ }
+ // Due to HDFS bug HDFS-8407, we can't distinguish between an error and empty
+ // folder, expscially for Kerberos enable setup, EAGAIN is quite common when
+ // the call is actually successful. Check again by Stat.
+ if (info == nullptr && errno != 0) {
+ FileStatistics stat;
+ TF_RETURN_IF_ERROR(Stat(dir, &stat));
}
- hdfs_->hdfsFreeFileInfo(info, entries);
if (entries > 0) {
return errors::FailedPrecondition("Cannot delete a non-empty directory.");