1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
|
# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Seq2seq layer operations for use in neural networks.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from tensorflow.contrib import layers
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import rnn
from tensorflow.python.ops import tensor_array_ops
from tensorflow.python.ops import variable_scope as vs
__all__ = ["dynamic_rnn_decoder"]
def dynamic_rnn_decoder(cell, decoder_fn, inputs=None, sequence_length=None,
parallel_iterations=None, swap_memory=False,
time_major=False, scope=None, name=None):
""" Dynamic RNN decoder for a sequence-to-sequence model specified by
RNNCell and decoder function.
The `dynamic_rnn_decoder` is similar to the `tf.python.ops.rnn.dynamic_rnn`
as the decoder does not make any assumptions of sequence length and batch
size of the input.
The `dynamic_rnn_decoder` has two modes: training or inference and expects
the user to create seperate functions for each.
Under both training and inference `cell` and `decoder_fn` is expected. Where
the `cell` performs computation at every timestep using the `raw_rnn` and
the `decoder_fn` allows modelling of early stopping, output, state, and next
input and context.
When training the user is expected to supply `inputs`. At every time step a
slice of the supplied input is fed to the `decoder_fn`, which modifies and
returns the input for the next time step.
`sequence_length` is needed at training time, i.e., when `inputs` is not
None, for dynamic unrolling. At test time, when `inputs` is None,
`sequence_length` is not needed.
Under inference `inputs` is expected to be `None` and the input is inferred
solely from the `decoder_fn`.
Args:
cell: An instance of RNNCell.
decoder_fn: A function that takes time, cell state, cell input,
cell output and context state. It returns a early stopping vector,
cell state, next input, cell output and context state.
Examples of decoder_fn can be found in the decoder_fn.py folder.
inputs: The inputs for decoding (embedded format).
If `time_major == False` (default), this must be a `Tensor` of shape:
`[batch_size, max_time, ...]`.
If `time_major == True`, this must be a `Tensor` of shape:
`[max_time, batch_size, ...]`.
The input to `cell` at each time step will be a `Tensor` with dimensions
`[batch_size, ...]`.
sequence_length: (optional) An int32/int64 vector sized `[batch_size]`.
if `inputs` is not None and `sequence_length` is None it is inferred
from the `inputs` as the maximal possible sequence length.
parallel_iterations: (Default: 32). The number of iterations to run in
parallel. Those operations which do not have any temporal dependency
and can be run in parallel, will be. This parameter trades off
time for space. Values >> 1 use more memory but take less time,
while smaller values use less memory but computations take longer.
swap_memory: Transparently swap the tensors produced in forward inference
but needed for back prop from GPU to CPU. This allows training RNNs
which would typically not fit on a single GPU, with very minimal (or no)
performance penalty.
time_major: The shape format of the `inputs` and `outputs` Tensors.
If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`.
If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`.
Using `time_major = True` is a bit more efficient because it avoids
transposes at the beginning and end of the RNN calculation. However,
most TensorFlow data is batch-major, so by default this function
accepts input and emits output in batch-major form.
scope: VariableScope for the `raw_rnn`;
defaults to None.
name: NameScope for the decoder;
defaults to "dynamic_rnn_decoder"
Returns:
A pair (outputs, state) where:
outputs: the RNN output 'Tensor'.
If time_major == False (default), this will be a `Tensor` shaped:
`[batch_size, max_time, cell.output_size]`.
If time_major == True, this will be a `Tensor` shaped:
`[max_time, batch_size, cell.output_size]`.
state: The final state and will be shaped
`[batch_size, cell.state_size]`.
Raises:
ValueError: if inputs is not None and has less than three dimensions.
"""
with ops.name_scope(name, "dynamic_rnn_decoder",
[cell, decoder_fn, inputs, sequence_length,
parallel_iterations, swap_memory, time_major, scope]):
if inputs is not None:
# Convert to tensor
inputs = ops.convert_to_tensor(inputs)
# Test input dimensions
if inputs.get_shape().ndims is not None and (
inputs.get_shape().ndims < 2):
raise ValueError("Inputs must have at least two dimensions")
# Setup of RNN (dimensions, sizes, length, initial state, dtype)
if not time_major:
# [batch, seq, features] -> [seq, batch, features]
inputs = array_ops.transpose(inputs, perm=[1, 0, 2])
dtype = inputs.dtype
# Get data input information
input_depth = int(inputs.get_shape()[2])
batch_depth = inputs.get_shape()[1].value
max_time = inputs.get_shape()[0].value
if max_time is None:
max_time = array_ops.shape(inputs)[0]
# Setup decoder inputs as TensorArray
inputs_ta = tensor_array_ops.TensorArray(dtype, size=max_time)
inputs_ta = inputs_ta.unpack(inputs)
def loop_fn(time, cell_output, cell_state, loop_state):
if cell_state is None: # first call, before while loop (in raw_rnn)
if cell_output is not None:
raise ValueError("Expected cell_output to be None when cell_state "
"is None, but saw: %s" % cell_output)
if loop_state is not None:
raise ValueError("Expected loop_state to be None when cell_state "
"is None, but saw: %s" % loop_state)
context_state = None
else: # subsequent calls, inside while loop, after cell excution
if isinstance(loop_state, tuple):
(done, context_state) = loop_state
else:
done = loop_state
context_state = None
# call decoder function
if inputs is not None: # training
# get next_cell_input
if cell_state is None:
next_cell_input = inputs_ta.read(0)
else:
if batch_depth is not None:
batch_size = batch_depth
else:
batch_size = array_ops.shape(done)[0]
next_cell_input = control_flow_ops.cond(
math_ops.equal(time, max_time),
lambda: array_ops.zeros([batch_size, input_depth], dtype=dtype),
lambda: inputs_ta.read(time))
(next_done, next_cell_state, next_cell_input, emit_output,
next_context_state) = decoder_fn(time, cell_state, next_cell_input,
cell_output, context_state)
else: # inference
# next_cell_input is obtained through decoder_fn
(next_done, next_cell_state, next_cell_input, emit_output,
next_context_state) = decoder_fn(time, cell_state, None, cell_output,
context_state)
# check if we are done
if next_done is None: # training
next_done = time >= sequence_length
# build next_loop_state
if next_context_state is None:
next_loop_state = next_done
else:
next_loop_state = (next_done, next_context_state)
return (next_done, next_cell_input, next_cell_state,
emit_output, next_loop_state)
# Run raw_rnn function
outputs_ta, state, _ = rnn.raw_rnn(
cell, loop_fn, parallel_iterations=parallel_iterations,
swap_memory=swap_memory, scope=scope)
outputs = outputs_ta.pack()
if not time_major:
# [seq, batch, features] -> [batch, seq, features]
outputs = array_ops.transpose(outputs, perm=[1, 0, 2])
return outputs, state
|
