Implement a file factory mechanism to handle network file systems.

- Env dispatches to a FileSystem interface
- FileSystemFactory is used to look up the correct FileSystem implementation
based on the prefix of the filename
- Provide a registration mechanism to register different factories
Change: 119268846

1 files changed, 243 insertions, 0 deletions

diff --git a/tensorflow/core/platform/file_system.h b/tensorflow/core/platform/file_system.h
new file mode 100644
index 0000000000..5fbd097ab2
--- /dev/null
+++ b/tensorflow/core/platform/file_system.h
@@ -0,0 +1,243 @@
+/* Copyright 2015 Google Inc. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_PLATFORM_FILE_SYSTEM_H_
+#define TENSORFLOW_CORE_PLATFORM_FILE_SYSTEM_H_
+
+#include <stdint.h>
+#include <functional>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/platform/macros.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/protobuf.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+
+class RandomAccessFile;
+class ReadOnlyMemoryRegion;
+class WritableFile;
+
+/// An generic interface for accessing a file system.
+class FileSystem {
+ public:
+  FileSystem() {}
+
+  virtual ~FileSystem();
+
+  /// The following functions are the implementations used by the corresponding
+  /// functions in the Env class.
+  virtual Status NewRandomAccessFile(const string& fname,
+                                     RandomAccessFile** result) = 0;
+
+  virtual Status NewWritableFile(const string& fname,
+                                 WritableFile** result) = 0;
+
+  virtual Status NewAppendableFile(const string& fname,
+                                   WritableFile** result) = 0;
+
+  virtual Status NewReadOnlyMemoryRegionFromFile(
+      const string& fname, ReadOnlyMemoryRegion** result) = 0;
+
+  virtual bool FileExists(const string& fname) = 0;
+
+  virtual Status GetChildren(const string& dir,
+                             std::vector<string>* result) = 0;
+
+  virtual Status DeleteFile(const string& fname) = 0;
+
+  virtual Status CreateDir(const string& dirname) = 0;
+
+  virtual Status DeleteDir(const string& dirname) = 0;
+
+  virtual Status GetFileSize(const string& fname, uint64* file_size) = 0;
+
+  virtual Status RenameFile(const string& src, const string& target) = 0;
+
+  // Translate an URI to a filename usable by the FileSystem implementation. The
+  // implementation in this class returns the name as-is.
+  virtual string TranslateName(const string& name) const;
+};
+
+// Degenerate file system that provides no implementations.
+class NullFileSystem : public FileSystem {
+ public:
+  NullFileSystem() {}
+
+  ~NullFileSystem() override = default;
+
+  Status NewRandomAccessFile(const string& fname,
+                             RandomAccessFile** result) override {
+    return errors::Unimplemented("NewRandomAccessFile unimplemented");
+  }
+
+  Status NewWritableFile(const string& fname, WritableFile** result) override {
+    return errors::Unimplemented("NewWritableFile unimplemented");
+  }
+
+  Status NewAppendableFile(const string& fname,
+                           WritableFile** result) override {
+    return errors::Unimplemented("NewAppendableFile unimplemented");
+  }
+
+  Status NewReadOnlyMemoryRegionFromFile(
+      const string& fname, ReadOnlyMemoryRegion** result) override {
+    return errors::Unimplemented(
+        "NewReadOnlyMemoryRegionFromFile unimplemented");
+  }
+
+  bool FileExists(const string& fname) override { return false; }
+
+  Status GetChildren(const string& dir, std::vector<string>* result) override {
+    return errors::Unimplemented("GetChildren unimplemented");
+  }
+
+  Status DeleteFile(const string& fname) override {
+    return errors::Unimplemented("DeleteFile unimplemented");
+  }
+
+  Status CreateDir(const string& dirname) override {
+    return errors::Unimplemented("CreateDir unimplemented");
+  }
+
+  Status DeleteDir(const string& dirname) override {
+    return errors::Unimplemented("DeleteDir unimplemented");
+  }
+
+  Status GetFileSize(const string& fname, uint64* file_size) override {
+    return errors::Unimplemented("GetFileSize unimplemented");
+  }
+
+  Status RenameFile(const string& src, const string& target) override {
+    return errors::Unimplemented("RenameFile unimplemented");
+  }
+};
+
+/// A file abstraction for randomly reading the contents of a file.
+class RandomAccessFile {
+ public:
+  RandomAccessFile() {}
+  virtual ~RandomAccessFile();
+
+  /// \brief Reads up to `n` bytes from the file starting at `offset`.
+  ///
+  /// `scratch[0..n-1]` may be written by this routine.  Sets `*result`
+  /// to the data that was read (including if fewer than `n` bytes were
+  /// successfully read).  May set `*result` to point at data in
+  /// `scratch[0..n-1]`, so `scratch[0..n-1]` must be live when
+  /// `*result` is used.
+  ///
+  /// On OK returned status: `n` bytes have been stored in `*result`.
+  /// On non-OK returned status: `[0..n]` bytes have been stored in `*result`.
+  ///
+  /// Returns `OUT_OF_RANGE` if fewer than n bytes were stored in `*result`
+  /// because of EOF.
+  ///
+  /// Safe for concurrent use by multiple threads.
+  virtual Status Read(uint64 offset, size_t n, StringPiece* result,
+                      char* scratch) const = 0;
+
+ private:
+  /// No copying allowed
+  RandomAccessFile(const RandomAccessFile&);
+  void operator=(const RandomAccessFile&);
+};
+
+/// \brief A file abstraction for sequential writing.
+///
+/// The implementation must provide buffering since callers may append
+/// small fragments at a time to the file.
+class WritableFile {
+ public:
+  WritableFile() {}
+  virtual ~WritableFile();
+
+  virtual Status Append(const StringPiece& data) = 0;
+  virtual Status Close() = 0;
+  virtual Status Flush() = 0;
+  virtual Status Sync() = 0;
+
+ private:
+  /// No copying allowed
+  WritableFile(const WritableFile&);
+  void operator=(const WritableFile&);
+};
+
+/// \brief A readonly memmapped file abstraction.
+///
+/// The implementation must guarantee that all memory is accessable when the
+/// object exists, independently from the Env that created it.
+class ReadOnlyMemoryRegion {
+ public:
+  ReadOnlyMemoryRegion() {}
+  virtual ~ReadOnlyMemoryRegion() = default;
+  virtual const void* data() = 0;
+  virtual uint64 length() = 0;
+};
+
+/// \brief A registry for file system implementations.
+///
+/// Filenames are specified as an URI, which is of the form
+/// [scheme://]<filename>.
+/// File system implementations are registered using the REGISTER_FILE_SYSTEM
+/// macro, providing the 'scheme' as the key.
+class FileSystemRegistry {
+ public:
+  typedef std::function<FileSystem*()> Factory;
+
+  virtual ~FileSystemRegistry();
+  virtual void Register(const string& scheme, Factory factory) = 0;
+  virtual FileSystem* Lookup(const string& scheme) = 0;
+};
+
+FileSystemRegistry* GlobalFileSystemRegistry();
+
+namespace register_file_system {
+
+template <typename Factory>
+struct Register {
+  Register(const string& scheme) {
+    ::tensorflow::GlobalFileSystemRegistry()->Register(
+        scheme, []() -> FileSystem* { return new Factory; });
+  }
+};
+
+}  // namespace register_file_system
+
+// Given URI of the form [scheme://]<filename>, return 'scheme'.
+string GetSchemeFromURI(const string& name);
+
+// Given URI of the form [scheme://]<filename>, return 'filename'.
+string GetNameFromURI(const string& name);
+
+}  // namespace tensorflow
+
+// Register a FileSystem implementation for a scheme. Files with names that have
+// "scheme://" prefixes are routed to use this implementation.
+#define REGISTER_FILE_SYSTEM(scheme, factory) \
+  REGISTER_FILE_SYSTEM_UNIQ_HELPER(__COUNTER__, scheme, factory)
+#define REGISTER_FILE_SYSTEM_UNIQ_HELPER(ctr, scheme, factory) \
+  REGISTER_FILE_SYSTEM_UNIQ(ctr, scheme, factory)
+#define REGISTER_FILE_SYSTEM_UNIQ(ctr, scheme, factory)        \
+  static ::tensorflow::register_file_system::Register<factory> \
+      register_ff##ctr TF_ATTRIBUTE_UNUSED =                   \
+          ::tensorflow::register_file_system::Register<factory>(scheme)
+
+#endif  // TENSORFLOW_CORE_PLATFORM_FILE_SYSTEM_H_