1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
|
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef TENSORFLOW_KERNELS_RELU_OP_FUNCTOR_H_
#define TENSORFLOW_KERNELS_RELU_OP_FUNCTOR_H_
// Functor definition for ReluOp and ReluGradOp, must be compilable by nvcc.
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#include "tensorflow/core/framework/tensor_types.h"
namespace tensorflow {
namespace functor {
// Functor used by ReluOp to do the computations.
template <typename Device, typename T>
struct Relu {
// Computes Relu activation.
//
// features: any shape.
// activations: same shape as "features".
void operator()(const Device& d, typename TTypes<T>::ConstTensor features,
typename TTypes<T>::Tensor activations) {
activations.device(d) = features.cwiseMax(static_cast<T>(0));
}
};
// Functor used by ReluGradOp to do the computations.
template <typename Device, typename T>
struct ReluGrad {
// Computes ReluGrad backprops.
//
// gradients: gradients backpropagated to the Relu op.
// features: either the inputs that were passed to the Relu or, or its
// outputs (using either one yields the same result here).
// backprops: gradients to backpropagate to the Relu inputs.
void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients,
typename TTypes<T>::ConstTensor features,
typename TTypes<T>::Tensor backprops) {
// NOTE: When the activation is exactly zero, we do not propagate the
// associated gradient value. This allows the output of the Relu to be used,
// as well as its input.
backprops.device(d) =
gradients * (features > static_cast<T>(0)).template cast<T>();
}
};
// Functor used by Relu6Op to do the computations.
template <typename Device, typename T>
struct Relu6 {
// Computes Relu6 activation.
//
// features: any shape.
// activations: same shape as "features".
void operator()(const Device& d, typename TTypes<T>::ConstTensor features,
typename TTypes<T>::Tensor activations) {
activations.device(d) =
features.cwiseMax(static_cast<T>(0)).cwiseMin(static_cast<T>(6));
}
};
// Functor used by ReluGradOp to do the computations.
template <typename Device, typename T>
struct Relu6Grad {
// Computes Relu6Grad backprops.
//
// gradients: gradients backpropagated to the Relu6 op.
// features: inputs that where passed to the Relu6 op.
// backprops: gradients to backpropagate to the Relu6 inputs.
void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients,
typename TTypes<T>::ConstTensor features,
typename TTypes<T>::Tensor backprops) {
// NOTE: When the activation is exactly zero or six, we
// arbitrarily choose to not propagate the associated gradient
// value.
backprops.device(d) =
gradients *
((features > static_cast<T>(0)) * (features < static_cast<T>(6)))
.template cast<T>();
}
};
// Functor used by EluOp to do the computations.
template <typename Device, typename T>
struct Elu {
// Computes Elu activation.
//
// features: any shape.
// activations: same shape as "features".
void operator()(const Device& d, typename TTypes<T>::ConstTensor features,
typename TTypes<T>::Tensor activations) {
// features.constant(?)
activations.device(d) =
(features < static_cast<T>(0))
.select(features.exp() - features.constant(static_cast<T>(1)),
features);
}
};
// Functor used by EluGradOp to do the computations.
template <typename Device, typename T>
struct EluGrad {
// Computes EluGrad backprops.
//
// gradients: gradients backpropagated to the Elu op.
// activations: outputs of the Elu op.
// backprops: gradients to backpropagate to the Elu inputs.
void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients,
typename TTypes<T>::ConstTensor activations,
typename TTypes<T>::Tensor backprops) {
backprops.device(d) =
(activations < static_cast<T>(0))
.select((activations + static_cast<T>(1)) * gradients, gradients);
}
};
// Functor used by SeluOp to do the computations.
template <typename Device, typename T>
struct Selu {
// Computes Selu activation.
//
// features: any shape.
// activations: same shape as "features".
void operator()(const Device& d, typename TTypes<T>::ConstTensor features,
typename TTypes<T>::Tensor activations) {
// features.constant(?)
const auto scale = static_cast<T>(1.0507009873554804934193349852946);
const auto scale_alpha = static_cast<T>(1.7580993408473768599402175208123);
const auto one = static_cast<T>(1);
const auto zero = static_cast<T>(0);
activations.device(d) =
(features < zero)
.select(scale_alpha * (features.exp() - features.constant(one)),
scale * features);
}
};
// Functor used by SeluGradOp to do the computations.
template <typename Device, typename T>
struct SeluGrad {
// Computes SeluGrad backprops.
//
// gradients: gradients backpropagated to the Selu op.
// activations: outputs of the Selu op.
// backprops: gradients to backpropagate to the Selu inputs.
void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients,
typename TTypes<T>::ConstTensor activations,
typename TTypes<T>::Tensor backprops) {
const auto scale = static_cast<T>(1.0507009873554804934193349852946);
const auto scale_alpha = static_cast<T>(1.7580993408473768599402175208123);
backprops.device(d) =
(activations < static_cast<T>(0)).select(
gradients * (activations + scale_alpha), gradients * scale);
}
};
} // namespace functor
} // namespace tensorflow
#endif // TENSORFLOW_KERNELS_RELU_OP_FUNCTOR_H_
|
