aboutsummaryrefslogtreecommitdiffhomepage
path: root/tensorflow/core/lib/io/format.cc
diff options
context:
space:
mode:
authorGravatar Manjunath Kudlur <keveman@gmail.com>2015-11-06 16:27:58 -0800
committerGravatar Manjunath Kudlur <keveman@gmail.com>2015-11-06 16:27:58 -0800
commitf41959ccb2d9d4c722fe8fc3351401d53bcf4900 (patch)
treeef0ca22cb2a5ac4bdec9d080d8e0788a53ed496d /tensorflow/core/lib/io/format.cc
TensorFlow: Initial commit of TensorFlow library.
TensorFlow is an open source software library for numerical computation using data flow graphs. Base CL: 107276108
Diffstat (limited to 'tensorflow/core/lib/io/format.cc')
-rw-r--r--tensorflow/core/lib/io/format.cc148
1 files changed, 148 insertions, 0 deletions
diff --git a/tensorflow/core/lib/io/format.cc b/tensorflow/core/lib/io/format.cc
new file mode 100644
index 0000000000..259cfc13dc
--- /dev/null
+++ b/tensorflow/core/lib/io/format.cc
@@ -0,0 +1,148 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "tensorflow/core/lib/io/format.h"
+
+#include "tensorflow/core/public/env.h"
+#include "tensorflow/core/lib/io/block.h"
+#include "tensorflow/core/lib/core/coding.h"
+#include "tensorflow/core/lib/hash/crc32c.h"
+#include "tensorflow/core/platform/port.h"
+#include "tensorflow/core/lib/core/errors.h"
+
+namespace tensorflow {
+namespace table {
+
+void BlockHandle::EncodeTo(string* dst) const {
+ // Sanity check that all fields have been set
+ assert(offset_ != ~static_cast<uint64>(0));
+ assert(size_ != ~static_cast<uint64>(0));
+ core::PutVarint64(dst, offset_);
+ core::PutVarint64(dst, size_);
+}
+
+Status BlockHandle::DecodeFrom(StringPiece* input) {
+ if (core::GetVarint64(input, &offset_) && core::GetVarint64(input, &size_)) {
+ return Status::OK();
+ } else {
+ return errors::DataLoss("bad block handle");
+ }
+}
+
+void Footer::EncodeTo(string* dst) const {
+#ifndef NDEBUG
+ const size_t original_size = dst->size();
+#endif
+ metaindex_handle_.EncodeTo(dst);
+ index_handle_.EncodeTo(dst);
+ dst->resize(2 * BlockHandle::kMaxEncodedLength); // Padding
+ core::PutFixed32(dst, static_cast<uint32>(kTableMagicNumber & 0xffffffffu));
+ core::PutFixed32(dst, static_cast<uint32>(kTableMagicNumber >> 32));
+ assert(dst->size() == original_size + kEncodedLength);
+}
+
+Status Footer::DecodeFrom(StringPiece* input) {
+ const char* magic_ptr = input->data() + kEncodedLength - 8;
+ const uint32 magic_lo = core::DecodeFixed32(magic_ptr);
+ const uint32 magic_hi = core::DecodeFixed32(magic_ptr + 4);
+ const uint64 magic =
+ ((static_cast<uint64>(magic_hi) << 32) | (static_cast<uint64>(magic_lo)));
+ if (magic != kTableMagicNumber) {
+ return errors::DataLoss("not an sstable (bad magic number)");
+ }
+
+ Status result = metaindex_handle_.DecodeFrom(input);
+ if (result.ok()) {
+ result = index_handle_.DecodeFrom(input);
+ }
+ if (result.ok()) {
+ // We skip over any leftover data (just padding for now) in "input"
+ const char* end = magic_ptr + 8;
+ *input = StringPiece(end, input->data() + input->size() - end);
+ }
+ return result;
+}
+
+Status ReadBlock(RandomAccessFile* file, const BlockHandle& handle,
+ BlockContents* result) {
+ result->data = StringPiece();
+ result->cachable = false;
+ result->heap_allocated = false;
+
+ // Read the block contents as well as the type/crc footer.
+ // See table_builder.cc for the code that built this structure.
+ size_t n = static_cast<size_t>(handle.size());
+ char* buf = new char[n + kBlockTrailerSize];
+ StringPiece contents;
+ Status s =
+ file->Read(handle.offset(), n + kBlockTrailerSize, &contents, buf);
+ if (!s.ok()) {
+ delete[] buf;
+ return s;
+ }
+ if (contents.size() != n + kBlockTrailerSize) {
+ delete[] buf;
+ return errors::DataLoss("truncated block read");
+ }
+
+ // Check the crc of the type and the block contents
+ const char* data = contents.data(); // Pointer to where Read put the data
+ // This checksum verification is optional. We leave it on for now
+ const bool verify_checksum = true;
+ if (verify_checksum) {
+ const uint32 crc = crc32c::Unmask(core::DecodeFixed32(data + n + 1));
+ const uint32 actual = crc32c::Value(data, n + 1);
+ if (actual != crc) {
+ delete[] buf;
+ s = errors::DataLoss("block checksum mismatch");
+ return s;
+ }
+ }
+
+ switch (data[n]) {
+ case kNoCompression:
+ if (data != buf) {
+ // File implementation gave us pointer to some other data.
+ // Use it directly under the assumption that it will be live
+ // while the file is open.
+ delete[] buf;
+ result->data = StringPiece(data, n);
+ result->heap_allocated = false;
+ result->cachable = false; // Do not double-cache
+ } else {
+ result->data = StringPiece(buf, n);
+ result->heap_allocated = true;
+ result->cachable = true;
+ }
+
+ // Ok
+ break;
+ case kSnappyCompression: {
+ size_t ulength = 0;
+ if (!port::Snappy_GetUncompressedLength(data, n, &ulength)) {
+ delete[] buf;
+ return errors::DataLoss("corrupted compressed block contents");
+ }
+ char* ubuf = new char[ulength];
+ if (!port::Snappy_Uncompress(data, n, ubuf)) {
+ delete[] buf;
+ delete[] ubuf;
+ return errors::DataLoss("corrupted compressed block contents");
+ }
+ delete[] buf;
+ result->data = StringPiece(ubuf, ulength);
+ result->heap_allocated = true;
+ result->cachable = true;
+ break;
+ }
+ default:
+ delete[] buf;
+ return errors::DataLoss("bad block type");
+ }
+
+ return Status::OK();
+}
+
+} // namespace table
+} // namespace tensorflow