1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
// See docs in ../ops/string_ops.cc.
#include <string>
#include "tensorflow/core/framework/kernel_def_builder.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/status.h"
#include "tensorflow/core/lib/strings/str_util.h"
namespace tensorflow {
namespace {
std::vector<string> Split(const string& str, const string& delimiter,
const bool skipEmpty) {
if (!delimiter.empty()) {
if (skipEmpty) {
return str_util::Split(str, delimiter, str_util::SkipEmpty());
}
return str_util::Split(str, delimiter);
}
std::vector<string> char_vector(str.size());
for (size_t i = 0; i < str.size(); ++i) {
char_vector[i] = str[i];
}
return char_vector;
}
} // namespace
class StringSplitOp : public OpKernel {
public:
explicit StringSplitOp(OpKernelConstruction* context)
: OpKernel(context), skip_empty_(true) {
bool skip_empty;
// By default skip_empty_ is true. We only get the value from attr if it is
// available, so that it is backward compatible.
if (context->GetAttr("skip_empty", &skip_empty).ok()) {
skip_empty_ = skip_empty;
}
}
void Compute(OpKernelContext* ctx) override {
const Tensor* input_tensor;
OP_REQUIRES_OK(ctx, ctx->input("input", &input_tensor));
OP_REQUIRES(ctx, TensorShapeUtils::IsVector(input_tensor->shape()),
errors::InvalidArgument("input must be a vector, got shape: ",
input_tensor->shape().DebugString()));
const auto input_vec = input_tensor->vec<string>();
const int64 batch_size = input_vec.dimension(0);
const Tensor* delimiter_tensor;
OP_REQUIRES_OK(ctx, ctx->input("delimiter", &delimiter_tensor));
OP_REQUIRES(
ctx, TensorShapeUtils::IsScalar(delimiter_tensor->shape()),
errors::InvalidArgument("delimiter must be a scalar, got shape: ",
delimiter_tensor->shape().DebugString()));
const auto delimiter_vec = delimiter_tensor->flat<string>();
const string& delimiter = delimiter_vec(0);
// Empty delimiter means split the input character by character.
std::vector<string> tokens;
// Guess that we'll be unpacking a handful of tokens per example.
static constexpr int kReserveSize = 4;
tokens.reserve(batch_size * kReserveSize);
int64 output_size = 0;
int64 max_num_entries = 0;
std::vector<int64> num_indices(batch_size);
for (int64 i = 0; i < batch_size; ++i) {
std::vector<string> parts = Split(input_vec(i), delimiter, skip_empty_);
int64 n_entries = parts.size();
num_indices[i] = n_entries;
output_size += n_entries;
max_num_entries = std::max(max_num_entries, n_entries);
tokens.insert(tokens.end(), parts.begin(), parts.end());
}
Tensor* sp_indices_t;
OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({output_size, 2}),
&sp_indices_t));
Tensor* sp_tokens_t;
OP_REQUIRES_OK(
ctx, ctx->allocate_output(1, TensorShape({output_size}), &sp_tokens_t));
Tensor* sp_shape_t;
OP_REQUIRES_OK(ctx, ctx->allocate_output(2, TensorShape({2}), &sp_shape_t));
auto sp_indices = sp_indices_t->matrix<int64>();
auto sp_tokens = sp_tokens_t->vec<string>();
auto sp_shape = sp_shape_t->vec<int64>();
sp_shape(0) = batch_size;
sp_shape(1) = max_num_entries;
size_t c = 0;
for (size_t i = 0; i < batch_size; ++i) {
for (size_t j = 0; j < num_indices[i]; ++j) {
sp_indices(c, 0) = i;
sp_indices(c, 1) = j;
sp_tokens(c) = tokens[c];
++c;
}
}
}
private:
bool skip_empty_;
};
REGISTER_KERNEL_BUILDER(Name("StringSplit").Device(DEVICE_CPU), StringSplitOp);
} // namespace tensorflow
|
