From f41959ccb2d9d4c722fe8fc3351401d53bcf4900 Mon Sep 17 00:00:00 2001 From: Manjunath Kudlur Date: Fri, 6 Nov 2015 16:27:58 -0800 Subject: TensorFlow: Initial commit of TensorFlow library. TensorFlow is an open source software library for numerical computation using data flow graphs. Base CL: 107276108 --- tensorflow/core/lib/io/format.cc | 148 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 tensorflow/core/lib/io/format.cc (limited to 'tensorflow/core/lib/io/format.cc') diff --git a/tensorflow/core/lib/io/format.cc b/tensorflow/core/lib/io/format.cc new file mode 100644 index 0000000000..259cfc13dc --- /dev/null +++ b/tensorflow/core/lib/io/format.cc @@ -0,0 +1,148 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "tensorflow/core/lib/io/format.h" + +#include "tensorflow/core/public/env.h" +#include "tensorflow/core/lib/io/block.h" +#include "tensorflow/core/lib/core/coding.h" +#include "tensorflow/core/lib/hash/crc32c.h" +#include "tensorflow/core/platform/port.h" +#include "tensorflow/core/lib/core/errors.h" + +namespace tensorflow { +namespace table { + +void BlockHandle::EncodeTo(string* dst) const { + // Sanity check that all fields have been set + assert(offset_ != ~static_cast(0)); + assert(size_ != ~static_cast(0)); + core::PutVarint64(dst, offset_); + core::PutVarint64(dst, size_); +} + +Status BlockHandle::DecodeFrom(StringPiece* input) { + if (core::GetVarint64(input, &offset_) && core::GetVarint64(input, &size_)) { + return Status::OK(); + } else { + return errors::DataLoss("bad block handle"); + } +} + +void Footer::EncodeTo(string* dst) const { +#ifndef NDEBUG + const size_t original_size = dst->size(); +#endif + metaindex_handle_.EncodeTo(dst); + index_handle_.EncodeTo(dst); + dst->resize(2 * BlockHandle::kMaxEncodedLength); // Padding + core::PutFixed32(dst, static_cast(kTableMagicNumber & 0xffffffffu)); + core::PutFixed32(dst, static_cast(kTableMagicNumber >> 32)); + assert(dst->size() == original_size + kEncodedLength); +} + +Status Footer::DecodeFrom(StringPiece* input) { + const char* magic_ptr = input->data() + kEncodedLength - 8; + const uint32 magic_lo = core::DecodeFixed32(magic_ptr); + const uint32 magic_hi = core::DecodeFixed32(magic_ptr + 4); + const uint64 magic = + ((static_cast(magic_hi) << 32) | (static_cast(magic_lo))); + if (magic != kTableMagicNumber) { + return errors::DataLoss("not an sstable (bad magic number)"); + } + + Status result = metaindex_handle_.DecodeFrom(input); + if (result.ok()) { + result = index_handle_.DecodeFrom(input); + } + if (result.ok()) { + // We skip over any leftover data (just padding for now) in "input" + const char* end = magic_ptr + 8; + *input = StringPiece(end, input->data() + input->size() - end); + } + return result; +} + +Status ReadBlock(RandomAccessFile* file, const BlockHandle& handle, + BlockContents* result) { + result->data = StringPiece(); + result->cachable = false; + result->heap_allocated = false; + + // Read the block contents as well as the type/crc footer. + // See table_builder.cc for the code that built this structure. + size_t n = static_cast(handle.size()); + char* buf = new char[n + kBlockTrailerSize]; + StringPiece contents; + Status s = + file->Read(handle.offset(), n + kBlockTrailerSize, &contents, buf); + if (!s.ok()) { + delete[] buf; + return s; + } + if (contents.size() != n + kBlockTrailerSize) { + delete[] buf; + return errors::DataLoss("truncated block read"); + } + + // Check the crc of the type and the block contents + const char* data = contents.data(); // Pointer to where Read put the data + // This checksum verification is optional. We leave it on for now + const bool verify_checksum = true; + if (verify_checksum) { + const uint32 crc = crc32c::Unmask(core::DecodeFixed32(data + n + 1)); + const uint32 actual = crc32c::Value(data, n + 1); + if (actual != crc) { + delete[] buf; + s = errors::DataLoss("block checksum mismatch"); + return s; + } + } + + switch (data[n]) { + case kNoCompression: + if (data != buf) { + // File implementation gave us pointer to some other data. + // Use it directly under the assumption that it will be live + // while the file is open. + delete[] buf; + result->data = StringPiece(data, n); + result->heap_allocated = false; + result->cachable = false; // Do not double-cache + } else { + result->data = StringPiece(buf, n); + result->heap_allocated = true; + result->cachable = true; + } + + // Ok + break; + case kSnappyCompression: { + size_t ulength = 0; + if (!port::Snappy_GetUncompressedLength(data, n, &ulength)) { + delete[] buf; + return errors::DataLoss("corrupted compressed block contents"); + } + char* ubuf = new char[ulength]; + if (!port::Snappy_Uncompress(data, n, ubuf)) { + delete[] buf; + delete[] ubuf; + return errors::DataLoss("corrupted compressed block contents"); + } + delete[] buf; + result->data = StringPiece(ubuf, ulength); + result->heap_allocated = true; + result->cachable = true; + break; + } + default: + delete[] buf; + return errors::DataLoss("bad block type"); + } + + return Status::OK(); +} + +} // namespace table +} // namespace tensorflow -- cgit v1.2.3