diff options
author | A. Unique TensorFlower <gardener@tensorflow.org> | 2016-08-25 14:28:26 -0800 |
---|---|---|
committer | TensorFlower Gardener <gardener@tensorflow.org> | 2016-08-25 20:16:17 -0700 |
commit | db7bdab6e586e02051556d9f36a7887500378cf9 (patch) | |
tree | 0285545a72591d7462b68d70f02ecbae69bdb1c7 /tensorflow/core/util/example_proto_fast_parsing.h | |
parent | 4501365d9a5bee11bbdc66154acecaf6897db946 (diff) |
Implement fast ParseExample.
Change: 131338565
Diffstat (limited to 'tensorflow/core/util/example_proto_fast_parsing.h')
-rw-r--r-- | tensorflow/core/util/example_proto_fast_parsing.h | 88 |
1 files changed, 88 insertions, 0 deletions
diff --git a/tensorflow/core/util/example_proto_fast_parsing.h b/tensorflow/core/util/example_proto_fast_parsing.h new file mode 100644 index 0000000000..6ed9d57838 --- /dev/null +++ b/tensorflow/core/util/example_proto_fast_parsing.h @@ -0,0 +1,88 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_TENSORFLOW_CORE_UTIL_EXAMPLE_PROTO_FAST_PARSING_H_ +#define THIRD_PARTY_TENSORFLOW_CORE_UTIL_EXAMPLE_PROTO_FAST_PARSING_H_ + +#include <string> +#include <unordered_map> +#include <vector> + +#include "tensorflow/core/example/example.pb.h" +#include "tensorflow/core/framework/allocator.h" +#include "tensorflow/core/framework/graph.pb.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/gtl/array_slice.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/sparse/sparse_tensor.h" + +namespace tensorflow { +namespace example { + +// FastParseExampleConfig defines how to parse features in Example. +// Each sub-config is responsible for one feature identified with feautre_name. +// FastParseExampleConfig can't have two sub-configs with the same feature_name. +// dtype identifies the type of output vector and the kind of Feature expected +// in Example. +struct FastParseExampleConfig { + struct Dense { + string feature_name; + DataType dtype; + // These 2 fields correspond exactly to dense_shapes and dense_defaults in + // ParseExample op. + // Documentation is avaliable in: tensorflow/core/ops/parsing_ops.cc + TensorShape shape; + Tensor default_value; + }; + + struct Sparse { + string feature_name; + DataType dtype; + }; + + std::vector<Dense> dense; + std::vector<Sparse> sparse; +}; + +// This is exactly the output of TF's ParseExample Op. +// Documentation is avaliable in: tensorflow/core/ops/parsing_ops.cc +struct Result { + std::vector<Tensor> sparse_indices; + std::vector<Tensor> sparse_values; + std::vector<Tensor> sparse_shapes; + std::vector<Tensor> dense_values; +}; + +// Parses a batch of serialized Example protos and converts them into result +// according to given config. +// Given example names have to either be empty or the same size as serialized. +// example_names are used only for error messages. +Status FastParseExample(const FastParseExampleConfig& config, + gtl::ArraySlice<string> serialized, + gtl::ArraySlice<string> example_names, + thread::ThreadPool* thread_pool, Result* result); + +// This function parses serialized Example and populates given example. +// It uses the same specialized parser as FastParseExample which is efficient. +// But then constructs Example which is relatively slow. +// It is exported here as a convenient API to test parser part separately. +bool TestFastParse(const string& serialized, Example* example); + +} // namespace example +} // namespace tensorflow + +#endif // THIRD_PARTY_TENSORFLOW_CORE_UTIL_EXAMPLE_PROTO_FAST_PARSING_H_ |