diff options
author | 2018-07-17 17:14:16 -0700 | |
---|---|---|
committer | 2018-07-17 17:17:57 -0700 | |
commit | 0c656ad9d7de201a7280c9b4ed561c40cbd9c5f3 (patch) | |
tree | 14e7a640cd7fa5b0e4815e5c51bc51e1f7d70297 /tensorflow/core/lib/io | |
parent | 36a66347e8e344cddee4a8d9123ccbcae40011b1 (diff) |
Automated rollback of commit 0ba51c741981c4f264dc06356a44b89ab9dbacd1
PiperOrigin-RevId: 205002413
Diffstat (limited to 'tensorflow/core/lib/io')
-rw-r--r-- | tensorflow/core/lib/io/zlib_compression_options.cc | 32 | ||||
-rw-r--r-- | tensorflow/core/lib/io/zlib_compression_options.h | 19 | ||||
-rw-r--r-- | tensorflow/core/lib/io/zlib_inputstream.cc | 109 | ||||
-rw-r--r-- | tensorflow/core/lib/io/zlib_inputstream.h | 27 |
4 files changed, 120 insertions, 67 deletions
diff --git a/tensorflow/core/lib/io/zlib_compression_options.cc b/tensorflow/core/lib/io/zlib_compression_options.cc new file mode 100644 index 0000000000..fc54083be1 --- /dev/null +++ b/tensorflow/core/lib/io/zlib_compression_options.cc @@ -0,0 +1,32 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/lib/io/zlib_compression_options.h" + +#include <zlib.h> + +namespace tensorflow { +namespace io { + +ZlibCompressionOptions::ZlibCompressionOptions() { + flush_mode = Z_NO_FLUSH; + window_bits = MAX_WBITS; + compression_level = Z_DEFAULT_COMPRESSION; + compression_method = Z_DEFLATED; + compression_strategy = Z_DEFAULT_STRATEGY; +} + +} // namespace io +} // namespace tensorflow diff --git a/tensorflow/core/lib/io/zlib_compression_options.h b/tensorflow/core/lib/io/zlib_compression_options.h index dc7218e866..238c1464fb 100644 --- a/tensorflow/core/lib/io/zlib_compression_options.h +++ b/tensorflow/core/lib/io/zlib_compression_options.h @@ -16,8 +16,6 @@ limitations under the License. #ifndef TENSORFLOW_LIB_IO_ZLIB_COMPRESSION_OPTIONS_H_ #define TENSORFLOW_LIB_IO_ZLIB_COMPRESSION_OPTIONS_H_ -#include <zlib.h> - #include "tensorflow/core/platform/types.h" namespace tensorflow { @@ -25,11 +23,14 @@ namespace io { class ZlibCompressionOptions { public: + ZlibCompressionOptions(); + static ZlibCompressionOptions DEFAULT(); static ZlibCompressionOptions RAW(); static ZlibCompressionOptions GZIP(); - int8 flush_mode = Z_NO_FLUSH; + // Defaults to Z_NO_FLUSH + int8 flush_mode; // Size of the buffer used for caching the data read from source file. int64 input_buffer_size = 256 << 10; @@ -71,7 +72,9 @@ class ZlibCompressionOptions { // window_bits value provided used while compressing. If a compressed stream // with a larger window size is given as input, inflate() will return with the // error code Z_DATA_ERROR instead of trying to allocate a larger window. - int8 window_bits = MAX_WBITS; + // + // Defaults to MAX_WBITS + int8 window_bits; // From the zlib manual (http://www.zlib.net/manual.html): // The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9: @@ -79,10 +82,10 @@ class ZlibCompressionOptions { // (the input data is simply copied a block at a time). Z_DEFAULT_COMPRESSION // requests a default compromise between speed and compression (currently // equivalent to level 6). - int8 compression_level = Z_DEFAULT_COMPRESSION; + int8 compression_level; - // The only one supported at this time. - int8 compression_method = Z_DEFLATED; + // Only Z_DEFLATED is supported at this time. + int8 compression_method; // From the zlib manual (http://www.zlib.net/manual.html): // The mem_level parameter specifies how much memory should be allocated for @@ -106,7 +109,7 @@ class ZlibCompressionOptions { // but not the correctness of the compressed output even if it is not set // appropriately. Z_FIXED prevents the use of dynamic Huffman codes, allowing // for a simpler decoder for special applications. - int8 compression_strategy = Z_DEFAULT_STRATEGY; + int8 compression_strategy; }; inline ZlibCompressionOptions ZlibCompressionOptions::DEFAULT() { diff --git a/tensorflow/core/lib/io/zlib_inputstream.cc b/tensorflow/core/lib/io/zlib_inputstream.cc index 47de36bf6c..d069db6d20 100644 --- a/tensorflow/core/lib/io/zlib_inputstream.cc +++ b/tensorflow/core/lib/io/zlib_inputstream.cc @@ -13,6 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include <zlib.h> + #include "tensorflow/core/lib/io/zlib_inputstream.h" #include "tensorflow/core/lib/strings/strcat.h" @@ -21,6 +23,35 @@ limitations under the License. namespace tensorflow { namespace io { +struct ZStreamDef { + ZStreamDef(size_t input_buffer_capacity, size_t output_buffer_capacity) + : input(new Bytef[input_buffer_capacity]), + output(new Bytef[output_buffer_capacity]), + stream(new z_stream) {} + + // Buffer for storing contents read from compressed stream. + // TODO(srbs): Consider using circular buffers. That would greatly simplify + // the implementation. + std::unique_ptr<Bytef[]> input; + + // Buffer for storing inflated contents of `input_stream_`. + std::unique_ptr<Bytef[]> output; + + // Configuration passed to `inflate`. + // + // z_stream_def_->stream->next_in: + // Next byte to de-compress. Points to some byte in + // z_stream_def_->streamdef_.input buffer. + // z_stream_def_->stream->avail_in: + // Number of bytes available to be decompressed at this time. + // z_stream_def_->stream->next_out: + // Next byte to write de-compressed data to. Points to some byte in + // z_stream_def_->streamdef_.output buffer. + // z_stream_def_->stream->avail_out: + // Number of free bytes available at write location. + std::unique_ptr<z_stream> stream; +}; + ZlibInputStream::ZlibInputStream( InputStreamInterface* input_stream, size_t input_buffer_bytes, // size of z_stream.next_in buffer @@ -30,10 +61,9 @@ ZlibInputStream::ZlibInputStream( input_stream_(input_stream), input_buffer_capacity_(input_buffer_bytes), output_buffer_capacity_(output_buffer_bytes), - z_stream_input_(new Bytef[input_buffer_capacity_]), - z_stream_output_(new Bytef[output_buffer_capacity_]), zlib_options_(zlib_options), - z_stream_(new z_stream), + z_stream_def_( + new ZStreamDef(input_buffer_capacity_, output_buffer_capacity_)), bytes_read_(0) { InitZlibBuffer(); } @@ -46,8 +76,8 @@ ZlibInputStream::ZlibInputStream(InputStreamInterface* input_stream, zlib_options, false) {} ZlibInputStream::~ZlibInputStream() { - if (z_stream_) { - inflateEnd(z_stream_.get()); + if (z_stream_def_->stream) { + inflateEnd(z_stream_def_->stream.get()); } if (owns_input_stream_) { delete input_stream_; @@ -56,51 +86,54 @@ ZlibInputStream::~ZlibInputStream() { Status ZlibInputStream::Reset() { TF_RETURN_IF_ERROR(input_stream_->Reset()); - inflateEnd(z_stream_.get()); + inflateEnd(z_stream_def_->stream.get()); InitZlibBuffer(); bytes_read_ = 0; return Status::OK(); } void ZlibInputStream::InitZlibBuffer() { - memset(z_stream_.get(), 0, sizeof(z_stream)); + memset(z_stream_def_->stream.get(), 0, sizeof(z_stream)); - z_stream_->zalloc = Z_NULL; - z_stream_->zfree = Z_NULL; - z_stream_->opaque = Z_NULL; - z_stream_->next_in = Z_NULL; - z_stream_->avail_in = 0; + z_stream_def_->stream->zalloc = Z_NULL; + z_stream_def_->stream->zfree = Z_NULL; + z_stream_def_->stream->opaque = Z_NULL; + z_stream_def_->stream->next_in = Z_NULL; + z_stream_def_->stream->avail_in = 0; - int status = inflateInit2(z_stream_.get(), zlib_options_.window_bits); + int status = + inflateInit2(z_stream_def_->stream.get(), zlib_options_.window_bits); CHECK_EQ(status, Z_OK) << "inflateInit failed with status " << status; - z_stream_->next_in = z_stream_input_.get(); - z_stream_->next_out = z_stream_output_.get(); - next_unread_byte_ = reinterpret_cast<char*>(z_stream_output_.get()); - z_stream_->avail_in = 0; - z_stream_->avail_out = output_buffer_capacity_; + z_stream_def_->stream->next_in = z_stream_def_->input.get(); + z_stream_def_->stream->next_out = z_stream_def_->output.get(); + next_unread_byte_ = reinterpret_cast<char*>(z_stream_def_->output.get()); + z_stream_def_->stream->avail_in = 0; + z_stream_def_->stream->avail_out = output_buffer_capacity_; } Status ZlibInputStream::ReadFromStream() { int bytes_to_read = input_buffer_capacity_; - char* read_location = reinterpret_cast<char*>(z_stream_input_.get()); + char* read_location = reinterpret_cast<char*>(z_stream_def_->input.get()); // If there are unread bytes in the input stream we move them to the head // of the stream to maximize the space available to read new data into. - if (z_stream_->avail_in > 0) { - uLong read_bytes = z_stream_->next_in - z_stream_input_.get(); + if (z_stream_def_->stream->avail_in > 0) { + uLong read_bytes = + z_stream_def_->stream->next_in - z_stream_def_->input.get(); // Remove `read_bytes` from the head of the input stream. // Move unread bytes to the head of the input stream. if (read_bytes > 0) { - memmove(z_stream_input_.get(), z_stream_->next_in, z_stream_->avail_in); + memmove(z_stream_def_->input.get(), z_stream_def_->stream->next_in, + z_stream_def_->stream->avail_in); } - bytes_to_read -= z_stream_->avail_in; - read_location += z_stream_->avail_in; + bytes_to_read -= z_stream_def_->stream->avail_in; + read_location += z_stream_def_->stream->avail_in; } string data; - // Try to read enough data to fill up z_stream_input_. + // Try to read enough data to fill up z_stream_def_->input. // TODO(rohanj): Add a char* version of ReadNBytes to InputStreamInterface // and use that instead to make this more efficient. Status s = input_stream_->ReadNBytes(bytes_to_read, &data); @@ -108,10 +141,10 @@ Status ZlibInputStream::ReadFromStream() { // Since we moved unread data to the head of the input stream we can point // next_in to the head of the input stream. - z_stream_->next_in = z_stream_input_.get(); + z_stream_def_->stream->next_in = z_stream_def_->input.get(); // Note: data.size() could be different from bytes_to_read. - z_stream_->avail_in += data.size(); + z_stream_def_->stream->avail_in += data.size(); if (!s.ok() && !errors::IsOutOfRange(s)) { return s; @@ -135,7 +168,8 @@ Status ZlibInputStream::ReadFromStream() { size_t ZlibInputStream::ReadBytesFromCache(size_t bytes_to_read, string* result) { size_t unread_bytes = - reinterpret_cast<char*>(z_stream_->next_out) - next_unread_byte_; + reinterpret_cast<char*>(z_stream_def_->stream->next_out) - + next_unread_byte_; size_t can_read_bytes = std::min(bytes_to_read, unread_bytes); if (can_read_bytes > 0) { result->append(next_unread_byte_, can_read_bytes); @@ -147,8 +181,9 @@ size_t ZlibInputStream::ReadBytesFromCache(size_t bytes_to_read, size_t ZlibInputStream::NumUnreadBytes() const { size_t read_bytes = - next_unread_byte_ - reinterpret_cast<char*>(z_stream_output_.get()); - return output_buffer_capacity_ - z_stream_->avail_out - read_bytes; + next_unread_byte_ - reinterpret_cast<char*>(z_stream_def_->output.get()); + return output_buffer_capacity_ - z_stream_def_->stream->avail_out - + read_bytes; } Status ZlibInputStream::ReadNBytes(int64 bytes_to_read, string* result) { @@ -167,14 +202,14 @@ Status ZlibInputStream::ReadNBytes(int64 bytes_to_read, string* result) { // completely consumed. This is an optimization and can be removed if // it causes problems. `ReadFromStream` is capable of handling partially // filled up buffers. - if (z_stream_->avail_in == 0) { + if (z_stream_def_->stream->avail_in == 0) { TF_RETURN_IF_ERROR(ReadFromStream()); } // Step 2. Setup output stream. - z_stream_->next_out = z_stream_output_.get(); - next_unread_byte_ = reinterpret_cast<char*>(z_stream_output_.get()); - z_stream_->avail_out = output_buffer_capacity_; + z_stream_def_->stream->next_out = z_stream_def_->output.get(); + next_unread_byte_ = reinterpret_cast<char*>(z_stream_def_->output.get()); + z_stream_def_->stream->avail_out = output_buffer_capacity_; // Step 3. Inflate Inflate Inflate! TF_RETURN_IF_ERROR(Inflate()); @@ -188,12 +223,12 @@ Status ZlibInputStream::ReadNBytes(int64 bytes_to_read, string* result) { int64 ZlibInputStream::Tell() const { return bytes_read_; } Status ZlibInputStream::Inflate() { - int error = inflate(z_stream_.get(), zlib_options_.flush_mode); + int error = inflate(z_stream_def_->stream.get(), zlib_options_.flush_mode); if (error != Z_OK && error != Z_STREAM_END) { string error_string = strings::StrCat("inflate() failed with error ", error); - if (z_stream_->msg != nullptr) { - strings::StrAppend(&error_string, ": ", z_stream_->msg); + if (z_stream_def_->stream->msg != nullptr) { + strings::StrAppend(&error_string, ": ", z_stream_def_->stream->msg); } return errors::DataLoss(error_string); } diff --git a/tensorflow/core/lib/io/zlib_inputstream.h b/tensorflow/core/lib/io/zlib_inputstream.h index 37339163ee..ac9e23ca97 100644 --- a/tensorflow/core/lib/io/zlib_inputstream.h +++ b/tensorflow/core/lib/io/zlib_inputstream.h @@ -16,8 +16,6 @@ limitations under the License. #ifndef TENSORFLOW_LIB_IO_ZLIB_INPUTSTREAM_H_ #define TENSORFLOW_LIB_IO_ZLIB_INPUTSTREAM_H_ -#include <zlib.h> - #include <string> #include "tensorflow/core/lib/core/status.h" @@ -30,6 +28,10 @@ limitations under the License. namespace tensorflow { namespace io { +// Forward declare some members of zlib.h, which is only included in the +// .cc file. +struct ZStreamDef; + // An ZlibInputStream provides support for reading from a stream compressed // using zlib (http://www.zlib.net/). Buffers the contents of the file. // @@ -79,28 +81,9 @@ class ZlibInputStream : public InputStreamInterface { size_t output_buffer_capacity_; // Size of z_stream_output_ char* next_unread_byte_; // Next unread byte in z_stream_output_ - // Buffer for storing contents read from compressed stream. - // TODO(srbs): Consider using circular buffers. That would greatly simplify - // the implementation. - std::unique_ptr<Bytef[]> z_stream_input_; - - // Buffer for storing inflated contents of `input_stream_`. - std::unique_ptr<Bytef[]> z_stream_output_; - ZlibCompressionOptions const zlib_options_; - // Configuration passed to `inflate`. - // - // z_stream_->next_in: - // Next byte to de-compress. Points to some byte in z_stream_input_ buffer. - // z_stream_->avail_in: - // Number of bytes available to be decompressed at this time. - // z_stream_->next_out: - // Next byte to write de-compressed data to. Points to some byte in - // z_stream_output_ buffer. - // z_stream_->avail_out: - // Number of free bytes available at write location. - std::unique_ptr<z_stream> z_stream_; + std::unique_ptr<ZStreamDef> z_stream_def_; // Reads data from `input_stream_` and tries to fill up `z_stream_input_` if // enough unread data is left in `input_stream_`. |