aboutsummaryrefslogtreecommitdiffhomepage
path: root/tensorflow/core/kernels
diff options
context:
space:
mode:
authorGravatar A. Unique TensorFlower <gardener@tensorflow.org>2017-08-15 12:08:29 -0700
committerGravatar TensorFlower Gardener <gardener@tensorflow.org>2017-08-15 12:12:05 -0700
commit28ce1d163eeffe618a6972c5245be0e660d94e85 (patch)
tree27e873a692b8bff17f1d6222bbe5c3e9689763ae /tensorflow/core/kernels
parent03a33c08dddbea5c58f7e04f8ab7ecae886f20bb (diff)
Merge changes from github.
END_PUBLIC --- Commit 9f81374c3 authored by raymondxyang<zihao.yang@microsoft.com> Committed by Rasmus Munk Larsen<rmlarsen@google.com>: Add option for build more python tests in Cmake (#11853) * Ignore Windows built project * Fix deprecated methods in tf.contrib.python * Fix regex match for Windows build in contrib.keras * Fix Regex match for Windows build in session_bundle * * Fix deprecated methods * Fix regex match for Windows * Fix compatibility issue with Python 3.x * Add missing ops into Windows build for test * Enabled more testcases for Windows build * Clean code and fix typo * Add conditional cmake mode for enabling more unit testcase * Add Cmake mode for major Contrib packages * Add supplementary info in RAEDME for new cmake option * * Update tf_tests after testing with TF 1.3 * Clean code and resolve conflicts * Fix unsafe regex matches and format code * Update exclude list after testing with latest master branch * Fix missing module --- Commit 98f0e1efe authored by Yong Tang<yong.tang.github@outlook.com> Committed by Rasmus Munk Larsen<rmlarsen@google.com>: Dynamic ksize and strides with MaxPool (#11875) * Dynamic ksize with max_pool This fix tries to fix the issue raised in 4746 where ksize is static (attr) with max_pool. This fix changes ksize to input tensor so that it is dynamic now. This fix fixes 4746. Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Add dynamic ksize to MaxPoolGrad and MaxPoolGradGrad Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Add test cases for max_pool_v2 Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Fix GPU Jenkins issue. Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Enable MaxPoolV2 in GPU Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Hide MaxPoolV2 and other fixes. Signed-off-by: Yong Tang <yong.tang.github@outlook.com> --- Commit 02d6bc185 authored by Bairen Yi<byronyi@users.noreply.github.com> Committed by Rasmus Munk Larsen<rmlarsen@google.com>: remove useless variable (#12212) --- Commit ed6b0d905 authored by namrata-ibm<bhavenamrata@gmail.com> Committed by Rasmus Munk Larsen<rmlarsen@google.com>: Adding support for s390x in calculation of cpu_frequency (#12201) --- Commit 627dfc9dd authored by Taehoon Lee<taehoonlee@snu.ac.kr> Committed by Taehoon Lee<taehoonlee@snu.ac.kr>: Fix typos --- Commit c0f9b0a91 authored by A. Unique TensorFlower<gardener@tensorflow.org> Committed by TensorFlower Gardener<gardener@tensorflow.org>: In fast-math mode emit a tanh that has a faster min/max. PiperOrigin-RevId: 164943597 --- Commit 87605f3d6 authored by Kay Zhu<kayzhu@google.com> Committed by TensorFlower Gardener<gardener@tensorflow.org>: [TF:XLA] Use HloEvaluator for ComputeConstant, remove the need of a dedicated compute constant backend. PiperOrigin-RevId: 164940970 --- Commit 881de45c2 authored by Taehoon Lee<me@taehoonlee.com> Committed by Rasmus Munk Larsen<rmlarsen@google.com>: Add bool type supports for GPU kernels (#11927) * Add bool type supports for GPU kernels * Add bool type test codes for GPU kernels --- Commit eeacdcdb1 authored by A. Unique TensorFlower<gardener@tensorflow.org> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Add missing "CPU" suffix in registrations. PiperOrigin-RevId: 164939527 --- Commit de01be952 authored by namrata-ibm<bhavenamrata@gmail.com> Committed by Rasmus Munk Larsen<rmlarsen@google.com>: Adding support for Big Endian in graph_constructor_test and wav_io (#12179) --- Commit 26719d29f authored by QingYing Chen<pkudysj@126.com> Committed by Rasmus Munk Larsen<rmlarsen@google.com>: Implement CRF decode (Viterbi decode) for tensor (#12056) * Implement CRF decoding for tensors * add test code for tensor version's CRF decoding * made modifications according to pylint * add some comments for crf decode * remove useless code * add comments at the top comment of crf module and add more comments in crf_test * capitalize first char of first word in comments * replace crf_decode test code with a deterministic example --- Commit f9a81ca2f authored by Pete Warden<pete@petewarden.com> Committed by gunan<gunan@google.com>: Create CI build script for Raspberry Pi (#12190) * Create CI build script for Raspberry Pi * Moved location of Pi build script --- Commit e2a163a90 authored by A. Unique TensorFlower<gardener@tensorflow.org> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Merge code from PR #11940 with internal changes from cl/164796436, and update Python tests to also run on GPU. PiperOrigin-RevId: 164929133 --- Commit 08bbfa187 authored by Taehoon Lee<me@taehoonlee.com> Committed by Rasmus Munk Larsen<rmlarsen@google.com>: Fix typos (#12195) --- Commit ab96f41fb authored by Luke Iwanski<luke@codeplay.com> Committed by Rasmus Munk Larsen<rmlarsen@google.com>: [OpenCL] Extends matmul_benchmark.py to cover SYCL (#11697) * [OpenCL] Extends matmul_benchmark.py to cover SYCL * Fixed typo * /gpu:0 -> /device:GPU:0 * Fixes control_flow_ops_py_test * /gpu: -> /device:GPU: * Fixes //tensorflow/python/profiler/internal:run_metadata_test * gpu: -> GPU: * Fixes tfprof_node * [OpenCL] Fixes device path to name with many colons (#123) The device path is constructed from a device name by replacing all colons with underscores. Some device names contain more than one colon, for example 'device:SYCL:0' which gives a path 'device_SYCL_0'. The previous code would not convert this back to the original device name, but rather to 'device:SYCL_0'. An alternative fix would be to convert all underscores to colons in the device name (i.e. remove the restriction inside `replace("_", ":", 1)`), however I'm not sure if there are any device names which contain underscores. * If no gpu device aviable fake one * gpu: -> device:GPU * Fixes profiler test * /gpu:x -> /device:GPU:x * Fixes debug_io_utils_test.cc test * Fixes device_name_utils_test.cc --- Commit 35e7a3665 authored by Yong Tang<yong.tang.github@outlook.com> Committed by Rasmus Munk Larsen<rmlarsen@google.com>: Remove unneeded casting of int64 for reverse_sequence (#12192) This fix remove unneeded cast of int64 for reverse_sequence: ``` lengths = math_ops.to_int64(lengths) ``` as int32 has already been enabled for reverse_sequence. Signed-off-by: Yong Tang <yong.tang.github@outlook.com> --- Commit 9fba8c185 authored by Anna R<annarev@google.com> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Add benchmark dashboard link to benchmarks doc. Also, I added a link and description for Benchmarks page to Community index page. PiperOrigin-RevId: 164924906 --- Commit bb6f32fa7 authored by Mark Heffernan<meheff@google.com> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Make HloAliasAnalysis updatable after changes to the HLO graph. As part of this change make HloAliasAnalysis a thinner layer which basically only holds a map from HloValue to HloBuffer and vice versa. PiperOrigin-RevId: 164923041 --- Commit 9103096c1 authored by A. Unique TensorFlower<gardener@tensorflow.org> Committed by Thomas K?ppe<tkoeppe@google.com>: Merged commit includes the following changes: 164923041 by meheff: Make HloAliasAnalysis updatable after changes to the HLO graph. As part of this change make HloAliasAnalysis a thinner layer which basically only holds a map from HloValue to HloBuffer and vice versa. -- PiperOrigin-RevId: 164923041 --- Commit 822603aed authored by A. Unique TensorFlower<gardener@tensorflow.org> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Merging sibling fusion instruction using multi_output_fusion PiperOrigin-RevId: 164920220 --- Commit c035aa2a8 authored by A. Unique TensorFlower<gardener@tensorflow.org> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 164917891 --- Commit e1e81d9ba authored by Luke Iwanski<luke@codeplay.com> Committed by Rasmus Munk Larsen<rmlarsen@google.com>: [OpenCL] Fixes double memcpy bug (#151) (#12173) * [OpenCL] Fixes double memcpy bug (#151) As the debg CopyOp is called on a Tensor without type, we need to use the DataType enum to get type information, and use this to pass the type on to Eigen. This is a workaround Eigen's need to have a type when calling memcpy. If the Eigen memcpy can be provided without a type requirement, then the memcpy in sycl_util is unnecessary. * Acts on feedback from: #12173/files/32cb12a9001b672425867b5a3110fd98e737a20b#r132496277 --- Commit d9ca2d86d authored by A. Unique TensorFlower<gardener@tensorflow.org> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Internal change PiperOrigin-RevId: 164916465 --- Commit b8d13d218 authored by A. Unique TensorFlower<gardener@tensorflow.org> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Remove more parts of DCASGD missed in the first pass. (47949b) PiperOrigin-RevId: 164914552 --- Commit 73b3d52c7 authored by Alexandre Passos<apassos@google.com> Committed by TensorFlower Gardener<gardener@tensorflow.org>: cmake fix PiperOrigin-RevId: 164911656 --- Commit 2173b5b0a authored by A. Unique TensorFlower<gardener@tensorflow.org> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Allow TFE_TensorHandleCopyToDevice to have the same device as src and destination. It will reuse the same underlying buffer in those cases. PiperOrigin-RevId: 164909906 --- Commit 13eb3b90e authored by Alexandre Passos<apassos@google.com> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Experimental C and Python APIs to invoke TensorFlow kernels on concrete values. PiperOrigin-RevId: 164902588 --- Commit 7dfabcc01 authored by A. Unique TensorFlower<gardener@tensorflow.org> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Initialize ExecutionOptions in ComputeConstant to default values. PiperOrigin-RevId: 164894867 --- Commit c8897e9bc authored by Benoit Steiner<bsteiner@google.com> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Static required time computation PiperOrigin-RevId: 164894645 --- Commit 076158f9b authored by A. Unique TensorFlower<gardener@tensorflow.org> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Enable implicit->explicit conversion by default. PiperOrigin-RevId: 164890915 --- Commit 58c4a4cb1 authored by A. Unique TensorFlower<gardener@tensorflow.org> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Bugfix: number of input channels is not necessarily in the last dimension, after introduction of data_format param. PiperOrigin-RevId: 164889729 --- Commit 8f9b1af8a authored by Igor Saprykin<isaprykin@google.com> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Recover MonitoredSession when the Coordinator is requested to stop with one of the _PREEMPTION_ERRORS. When SyncReplicasOptimizer is used, a preemption in the Coordinator may result in two cases: Case 1) the session gets silently marked as complete Case 2) the session gets stuck This CL aims to solve and verify solutions for both of these problems. Fix 1 changes the should_stop logic. Fix 2 changes the CoordinatedSession.run() logic. SyncReplicasOptimizer runs a separate set of threads using a Coordinator instance. Those threads do FIFOQueue.enqueue; the main thread does a blocking FIFOQueue.dequeue. `sync_token_q` FIFOQueue is on parameter-servers. When one of the PS instances gets preempted, an AbortedError causes the Coordinator to stop via request_stop(ex). That by itself changes the state of MonitoredSession.should_stop() to True (Fix 1). Results of the blocking Dequeue operation are sent to the chief worker via Recv. What happens next depends on the amount of tokens in `sync_token_q`. If there are enough for the next call to Dequeue to return, then the low-level "tf session run() call" returns. The next iteration of the `while not MonitoredSession.should_stop()` loop decides that the training is complete (Case 1). If there are not enough tokens in `sync_token_q`, then the blocking Dequeue is going to keep waiting for them. This results in the graph execution getting stuck and the whole session getting garbage collected after 10 minutes (Case 2). We decided to fix that by re-creating a session after it gets garbage collected (Fix 2). An alternative was to try to cancel the pending Dequeue operation, but it's not clear that it is the right thing to do and it is also not easy. PiperOrigin-RevId: 164888390 --- Commit 46e4de6e5 authored by A. Unique TensorFlower<gardener@tensorflow.org> Committed by TensorFlower Gardener<gardener@tensorflow.org>: Undo loop fusion changes for now as they seem to be altering a few results. END_PUBLIC RELNOTES: n/a BEGIN_PUBLIC BEGIN_PUBLIC Automated g4 rollback of changelist 164825735 PiperOrigin-RevId: 165340331
Diffstat (limited to 'tensorflow/core/kernels')
-rw-r--r--tensorflow/core/kernels/BUILD1
-rw-r--r--tensorflow/core/kernels/bias_op.cc55
-rw-r--r--tensorflow/core/kernels/concat_lib_gpu.cc1
-rw-r--r--tensorflow/core/kernels/concat_lib_gpu_impl.cu.cc4
-rw-r--r--tensorflow/core/kernels/concat_op.cc1
-rw-r--r--tensorflow/core/kernels/debug_ops.h7
-rw-r--r--tensorflow/core/kernels/maxpooling_op.cc450
-rw-r--r--tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc65
-rw-r--r--tensorflow/core/kernels/mkl_conv_ops.cc45
-rw-r--r--tensorflow/core/kernels/mkl_tfconv_op.cc38
-rw-r--r--tensorflow/core/kernels/pack_op.cc1
-rw-r--r--tensorflow/core/kernels/pooling_ops_common.h215
-rw-r--r--tensorflow/core/kernels/qr_op_impl.h4
-rw-r--r--tensorflow/core/kernels/reshape_op.cc1
-rw-r--r--tensorflow/core/kernels/sparse_matmul_op.h26
-rw-r--r--tensorflow/core/kernels/tile_ops.cc12
-rw-r--r--tensorflow/core/kernels/unique_op.cc62
-rw-r--r--tensorflow/core/kernels/unpack_op.cc3
18 files changed, 774 insertions, 217 deletions
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index a5e3a5feea..05974f5a90 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -103,7 +103,6 @@ tf_kernel_library(
"strided_slice_op.h",
"strided_slice_op_impl.h",
"strided_slice_op_gpu.cu.cc",
- "slice_op_gpu.cu.cc",
],
deps = [
":bounds_check",
diff --git a/tensorflow/core/kernels/bias_op.cc b/tensorflow/core/kernels/bias_op.cc
index 10f5d4ce85..b3a77d1caa 100644
--- a/tensorflow/core/kernels/bias_op.cc
+++ b/tensorflow/core/kernels/bias_op.cc
@@ -35,14 +35,13 @@ namespace tensorflow {
typedef Eigen::ThreadPoolDevice CPUDevice;
typedef Eigen::GpuDevice GPUDevice;
+#ifdef TENSORFLOW_USE_SYCL
+typedef Eigen::SyclDevice SYCLDevice;
+#endif // TENSORFLOW_USE_SYCL
template <typename Device, typename T>
-class BiasOp;
-
-template <typename T>
-class BiasOp<CPUDevice, T> : public BinaryOp<T> {
+class BiasOp : public BinaryOp<T> {
public:
- typedef CPUDevice Device;
explicit BiasOp(OpKernelConstruction* context) : BinaryOp<T>(context) {
string data_format;
if (context->GetAttr("data_format", &data_format).ok()) {
@@ -52,7 +51,8 @@ class BiasOp<CPUDevice, T> : public BinaryOp<T> {
data_format_ = FORMAT_NHWC;
}
OP_REQUIRES(context, data_format_ == FORMAT_NHWC,
- errors::InvalidArgument("CPU BiasOp only supports NHWC."));
+ errors::InvalidArgument(context->device()->attributes().name() +
+ " BiasOp only supports NHWC."));
}
void Compute(OpKernelContext* context) override {
@@ -122,6 +122,21 @@ class BiasOp<CPUDevice, T> : public BinaryOp<T> {
TF_CALL_NUMBER_TYPES(REGISTER_KERNEL);
#undef REGISTER_KERNEL
+#ifdef TENSORFLOW_USE_SYCL
+#define REGISTER_KERNEL(type) \
+ REGISTER_KERNEL_BUILDER( \
+ Name("BiasAdd").Device(DEVICE_SYCL).TypeConstraint<type>("T"), \
+ BiasOp<SYCLDevice, type>); \
+ REGISTER_KERNEL_BUILDER( \
+ Name("BiasAddV1").Device(DEVICE_SYCL).TypeConstraint<type>("T"), \
+ BiasOp<SYCLDevice, type>);
+
+TF_CALL_INTEGRAL_TYPES(REGISTER_KERNEL);
+REGISTER_KERNEL(float);
+REGISTER_KERNEL(double);
+#undef REGISTER_KERNEL
+#endif // TENSORFLOW_USE_SYCL
+
namespace {
void GetBiasValueDims(const Tensor& value_tensor, TensorFormat data_format,
@@ -165,12 +180,8 @@ struct AccumulatorType<Eigen::half> {
} // namespace
template <typename Device, typename T>
-class BiasGradOp;
-
-template <typename T>
-class BiasGradOp<CPUDevice, T> : public OpKernel {
+class BiasGradOp : public OpKernel {
public:
- typedef CPUDevice Device;
explicit BiasGradOp(OpKernelConstruction* context) : OpKernel(context) {
string data_format;
if (context->GetAttr("data_format", &data_format).ok()) {
@@ -180,7 +191,8 @@ class BiasGradOp<CPUDevice, T> : public OpKernel {
data_format_ = FORMAT_NHWC;
}
OP_REQUIRES(context, data_format_ == FORMAT_NHWC,
- errors::InvalidArgument("CPU BiasGradOp only supports NHWC."));
+ errors::InvalidArgument(context->device()->attributes().name() +
+ " BiasGradOp only supports NHWC."));
}
void Compute(OpKernelContext* context) override {
@@ -192,8 +204,9 @@ class BiasGradOp<CPUDevice, T> : public OpKernel {
output_backprop.shape().DebugString()));
OP_REQUIRES(
- context, FastBoundsCheck(output_backprop.NumElements(),
- std::numeric_limits<int32>::max()),
+ context,
+ FastBoundsCheck(output_backprop.NumElements(),
+ std::numeric_limits<int32>::max()),
errors::InvalidArgument("BiasGrad requires tensor size <= int32 max"));
int32 batch, height, width, channel;
@@ -215,7 +228,7 @@ class BiasGradOp<CPUDevice, T> : public OpKernel {
#else
Eigen::array<int, 1> reduction_axis = {0};
#endif
- output->template flat<T>().device(context->eigen_device<CPUDevice>()) =
+ output->template flat<T>().device(context->eigen_device<Device>()) =
output_backprop.flat<T>()
.template cast<typename AccumulatorType<T>::type>()
.reshape(two_dims)
@@ -237,6 +250,18 @@ class BiasGradOp<CPUDevice, T> : public OpKernel {
TF_CALL_NUMBER_TYPES(REGISTER_KERNEL);
#undef REGISTER_KERNEL
+#ifdef TENSORFLOW_USE_SYCL
+#define REGISTER_KERNEL(type) \
+ REGISTER_KERNEL_BUILDER( \
+ Name("BiasAddGrad").Device(DEVICE_SYCL).TypeConstraint<type>("T"), \
+ BiasGradOp<SYCLDevice, type>);
+
+TF_CALL_INTEGRAL_TYPES(REGISTER_KERNEL);
+REGISTER_KERNEL(float);
+REGISTER_KERNEL(double);
+#undef REGISTER_KERNEL
+#endif // TENSORFLOW_USE_SYCL
+
#if GOOGLE_CUDA
template <typename T>
class BiasOp<GPUDevice, T> : public BinaryOp<T> {
diff --git a/tensorflow/core/kernels/concat_lib_gpu.cc b/tensorflow/core/kernels/concat_lib_gpu.cc
index 5159cdaa6e..319ead49ef 100644
--- a/tensorflow/core/kernels/concat_lib_gpu.cc
+++ b/tensorflow/core/kernels/concat_lib_gpu.cc
@@ -117,6 +117,7 @@ TF_CALL_complex64(REGISTER);
TF_CALL_complex128(REGISTER);
TF_CALL_int64(REGISTER);
REGISTER(bfloat16);
+REGISTER(bool);
#undef REGISTER
diff --git a/tensorflow/core/kernels/concat_lib_gpu_impl.cu.cc b/tensorflow/core/kernels/concat_lib_gpu_impl.cu.cc
index f971637d5d..0f7adaf24a 100644
--- a/tensorflow/core/kernels/concat_lib_gpu_impl.cu.cc
+++ b/tensorflow/core/kernels/concat_lib_gpu_impl.cu.cc
@@ -203,24 +203,28 @@ TF_CALL_complex64(REGISTER_GPUCONCAT32);
TF_CALL_complex128(REGISTER_GPUCONCAT32);
TF_CALL_int64(REGISTER_GPUCONCAT32);
REGISTER_GPUCONCAT32(bfloat16);
+REGISTER_GPUCONCAT32(bool);
TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPUCONCAT64);
TF_CALL_complex64(REGISTER_GPUCONCAT64);
TF_CALL_complex128(REGISTER_GPUCONCAT64);
TF_CALL_int64(REGISTER_GPUCONCAT64);
REGISTER_GPUCONCAT64(bfloat16);
+REGISTER_GPUCONCAT64(bool);
TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU32);
TF_CALL_complex64(REGISTER_GPU32);
TF_CALL_complex128(REGISTER_GPU32);
TF_CALL_int64(REGISTER_GPU32);
REGISTER_GPU32(bfloat16);
+REGISTER_GPU32(bool);
TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU64);
TF_CALL_complex64(REGISTER_GPU64);
TF_CALL_complex128(REGISTER_GPU64);
TF_CALL_int64(REGISTER_GPU64);
REGISTER_GPU64(bfloat16);
+REGISTER_GPU64(bool);
#undef REGISTER_GPUCONCAT32
#undef REGISTER_GPUCONCAT64
diff --git a/tensorflow/core/kernels/concat_op.cc b/tensorflow/core/kernels/concat_op.cc
index 01a744dc7e..8e480aa995 100644
--- a/tensorflow/core/kernels/concat_op.cc
+++ b/tensorflow/core/kernels/concat_op.cc
@@ -196,6 +196,7 @@ REGISTER_GPU(bfloat16);
TF_CALL_complex64(REGISTER_GPU);
TF_CALL_complex128(REGISTER_GPU);
TF_CALL_int64(REGISTER_GPU);
+REGISTER_GPU(bool);
#undef REGISTER_GPU
// A special GPU kernel for int32.
diff --git a/tensorflow/core/kernels/debug_ops.h b/tensorflow/core/kernels/debug_ops.h
index ef12e2e42c..2c21053121 100644
--- a/tensorflow/core/kernels/debug_ops.h
+++ b/tensorflow/core/kernels/debug_ops.h
@@ -94,12 +94,7 @@ class CopyOp : public OpKernel {
!context->input_alloc_attr(0).on_host();
if (off_host_input) {
- auto size = src_tensor.NumElements() * sizeof(src_tensor.dtype());
- auto dst_ptr = GetBase(copied_tensor);
- auto src_ptr = GetBase(&src_tensor);
- typedef decltype(src_tensor.dtype()) ttype;
- context->eigen_sycl_device().memcpy(
- dst_ptr, static_cast<const ttype*>(src_ptr), size);
+ SYCLmemcpy(context->eigen_sycl_device(), src_tensor, copied_tensor);
} else {
*copied_tensor = tensor::DeepCopy(src_tensor);
}
diff --git a/tensorflow/core/kernels/maxpooling_op.cc b/tensorflow/core/kernels/maxpooling_op.cc
index 6cb56797bf..8d825c13d7 100644
--- a/tensorflow/core/kernels/maxpooling_op.cc
+++ b/tensorflow/core/kernels/maxpooling_op.cc
@@ -208,22 +208,26 @@ class MaxPoolingGradOp : public OpKernel {
errors::InvalidArgument("Default MaxPoolingGradOp only supports NHWC ",
"on device type ",
DeviceTypeString(context->device_type())));
- OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
- OP_REQUIRES(context, ksize_.size() == 4,
- errors::InvalidArgument("Sliding window ksize field must "
- "specify 4 dimensions"));
- OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
- OP_REQUIRES(context, stride_.size() == 4,
- errors::InvalidArgument("Sliding window strides field must "
- "specify 4 dimensions"));
+
+ if (context->num_inputs() == 3) {
+ OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
+ OP_REQUIRES(context, ksize_.size() == 4,
+ errors::InvalidArgument("Sliding window ksize field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
+ OP_REQUIRES(context, stride_.size() == 4,
+ errors::InvalidArgument("Sliding window strides field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES(context, ksize_[0] == 1 && stride_[0] == 1,
+ errors::Unimplemented(
+ "Pooling is not yet supported on the batch dimension."));
+ OP_REQUIRES(
+ context, ksize_[3] == 1 && stride_[3] == 1,
+ errors::Unimplemented(
+ "MaxPoolingGrad is not yet supported on the depth dimension."));
+ }
+
OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
- OP_REQUIRES(context, ksize_[0] == 1 && stride_[0] == 1,
- errors::Unimplemented(
- "Pooling is not yet supported on the batch dimension."));
- OP_REQUIRES(
- context, ksize_[3] == 1 && stride_[3] == 1,
- errors::Unimplemented(
- "MaxPoolingGrad is not yet supported on the depth dimension."));
}
void Compute(OpKernelContext* context) override {
@@ -250,8 +254,35 @@ class MaxPoolingGradOp : public OpKernel {
OP_REQUIRES_OK(context, context->allocate_temp(DataTypeToEnum<int64>::v(),
tensor_out.shape(),
&tensor_out_arg_max));
+ std::vector<int32> ksize = ksize_;
+ std::vector<int32> stride = stride_;
+ if (context->num_inputs() == 5) {
+ const Tensor& tensor_ksize = context->input(3);
+ auto value_ksize = tensor_ksize.flat<int32>();
+ ksize.resize(tensor_ksize.shape().num_elements());
+ std::copy_n(&value_ksize(0), ksize.size(), ksize.begin());
+
+ const Tensor& tensor_stride = context->input(4);
+ auto value_stride = tensor_stride.flat<int32>();
+ stride.resize(tensor_stride.shape().num_elements());
+ std::copy_n(&value_stride(0), stride.size(), stride.begin());
+ }
- PoolParameters params{context, ksize_, stride_,
+ OP_REQUIRES(context, ksize.size() == 4,
+ errors::InvalidArgument("Sliding window ksize field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES(context, stride.size() == 4,
+ errors::InvalidArgument("Sliding window strides field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES(context, ksize[0] == 1 && stride[0] == 1,
+ errors::Unimplemented(
+ "Pooling is not yet supported on the batch dimension."));
+ OP_REQUIRES(
+ context, ksize[3] == 1 && stride[3] == 1,
+ errors::Unimplemented(
+ "MaxPoolingGrad is not yet supported on the depth dimension."));
+
+ PoolParameters params{context, ksize, stride,
padding_, FORMAT_NHWC, tensor_in.shape()};
if (!context->status().ok()) {
return;
@@ -309,20 +340,22 @@ class MaxPoolingGradOp<Eigen::GpuDevice, T> : public OpKernel {
OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format));
OP_REQUIRES(context, FormatFromString(data_format, &data_format_),
errors::InvalidArgument("Invalid data format"));
- OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
- OP_REQUIRES(context, ksize_.size() == 4,
- errors::InvalidArgument("Sliding window ksize field must "
- "specify 4 dimensions"));
- OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
- OP_REQUIRES(context, stride_.size() == 4,
- errors::InvalidArgument("Sliding window strides field must "
- "specify 4 dimensions"));
+ if (context->num_inputs() == 3) {
+ OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
+ OP_REQUIRES(context, ksize_.size() == 4,
+ errors::InvalidArgument("Sliding window ksize field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
+ OP_REQUIRES(context, stride_.size() == 4,
+ errors::InvalidArgument("Sliding window strides field must "
+ "specify 4 dimensions"));
+ const int32 ksize_n = GetTensorDim(ksize_, data_format_, 'N');
+ const int32 stride_n = GetTensorDim(stride_, data_format_, 'N');
+ OP_REQUIRES(context, ksize_n == 1 && stride_n == 1,
+ errors::Unimplemented(
+ "Pooling is not yet supported on the batch dimension."));
+ }
OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
- const int32 ksize_n = GetTensorDim(ksize_, data_format_, 'N');
- const int32 stride_n = GetTensorDim(stride_, data_format_, 'N');
- OP_REQUIRES(context, ksize_n == 1 && stride_n == 1,
- errors::Unimplemented(
- "Pooling is not yet supported on the batch dimension."));
use_dnn_ = CanUseCudnn();
}
@@ -343,15 +376,40 @@ class MaxPoolingGradOp<Eigen::GpuDevice, T> : public OpKernel {
TensorShape output_shape = tensor_in.shape();
+ std::vector<int32> ksize = ksize_;
+ std::vector<int32> stride = stride_;
+ if (context->num_inputs() == 5) {
+ const Tensor& tensor_ksize = context->input(3);
+ auto value_ksize = tensor_ksize.flat<int32>();
+ ksize.resize(tensor_ksize.shape().num_elements());
+ std::copy_n(&value_ksize(0), ksize.size(), ksize.begin());
+
+ const Tensor& tensor_stride = context->input(4);
+ auto value_stride = tensor_stride.flat<int32>();
+ stride.resize(tensor_stride.shape().num_elements());
+ std::copy_n(&value_stride(0), stride.size(), stride.begin());
+ }
+ OP_REQUIRES(context, ksize.size() == 4,
+ errors::InvalidArgument("Sliding window ksize field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES(context, stride.size() == 4,
+ errors::InvalidArgument("Sliding window strides field must "
+ "specify 4 dimensions"));
+ const int32 ksize_n = GetTensorDim(ksize, data_format_, 'N');
+ const int32 stride_n = GetTensorDim(stride, data_format_, 'N');
+ OP_REQUIRES(context, ksize_n == 1 && stride_n == 1,
+ errors::Unimplemented(
+ "Pooling is not yet supported on the batch dimension."));
+
if (use_dnn_) {
DnnPoolingGradOp<T>::Compute(
- context, perftools::gputools::dnn::PoolingMode::kMaximum, ksize_,
- stride_, padding_, data_format_, &tensor_in, &tensor_out,
- out_backprop, output_shape);
+ context, perftools::gputools::dnn::PoolingMode::kMaximum, ksize,
+ stride, padding_, data_format_, &tensor_in, &tensor_out, out_backprop,
+ output_shape);
} else {
CHECK(data_format_ == FORMAT_NHWC)
<< "Non-Cudnn MaxPoolGrad only supports NHWC format";
- MaxPoolingBackwardCustomKernel<T>(context, ksize_, stride_, padding_,
+ MaxPoolingBackwardCustomKernel<T>(context, ksize, stride, padding_,
&tensor_in, out_backprop, output_shape);
}
}
@@ -386,22 +444,25 @@ class MaxPoolingGradGradOp : public OpKernel {
errors::InvalidArgument(
"Default MaxPoolingGradGradOp only supports NHWC ",
"on device type ", DeviceTypeString(context->device_type())));
- OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
- OP_REQUIRES(context, ksize_.size() == 4,
- errors::InvalidArgument("Sliding window ksize field must "
- "specify 4 dimensions"));
- OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
- OP_REQUIRES(context, stride_.size() == 4,
- errors::InvalidArgument("Sliding window strides field must "
- "specify 4 dimensions"));
+
OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
- OP_REQUIRES(context, ksize_[0] == 1 && stride_[0] == 1,
- errors::Unimplemented(
- "Pooling is not yet supported on the batch dimension."));
- OP_REQUIRES(
- context, ksize_[3] == 1 && stride_[3] == 1,
- errors::Unimplemented(
- "MaxPoolingGradGrad is not yet supported on the depth dimension."));
+
+ if (context->num_inputs() == 3) {
+ OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
+ OP_REQUIRES(context, ksize_.size() == 4,
+ errors::InvalidArgument("Sliding window ksize field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
+ OP_REQUIRES(context, stride_.size() == 4,
+ errors::InvalidArgument("Sliding window strides field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES(context, ksize_[0] == 1 && stride_[0] == 1,
+ errors::Unimplemented(
+ "Pooling is not yet supported on the batch dimension."));
+ OP_REQUIRES(context, ksize_[3] == 1 && stride_[3] == 1,
+ errors::Unimplemented("MaxPoolingGradGrad is not yet "
+ "supported on the depth dimension."));
+ }
}
void Compute(OpKernelContext* context) override {
@@ -419,7 +480,35 @@ class MaxPoolingGradGradOp : public OpKernel {
context, out_grad_backprop.dims() == 4,
errors::InvalidArgument("out_grad_backprop must be 4-dimensional"));
- PoolParameters params{context, ksize_, stride_,
+ std::vector<int32> ksize = ksize_;
+ std::vector<int32> stride = stride_;
+ if (context->num_inputs() == 5) {
+ const Tensor& tensor_ksize = context->input(3);
+ auto value_ksize = tensor_ksize.flat<int32>();
+ ksize.resize(tensor_ksize.shape().num_elements());
+ std::copy_n(&value_ksize(0), ksize.size(), ksize.begin());
+
+ const Tensor& tensor_stride = context->input(4);
+ auto value_stride = tensor_stride.flat<int32>();
+ stride.resize(tensor_stride.shape().num_elements());
+ std::copy_n(&value_stride(0), stride.size(), stride.begin());
+ }
+
+ OP_REQUIRES(context, ksize.size() == 4,
+ errors::InvalidArgument("Sliding window ksize field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES(context, stride.size() == 4,
+ errors::InvalidArgument("Sliding window strides field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES(context, ksize[0] == 1 && stride[0] == 1,
+ errors::Unimplemented(
+ "Pooling is not yet supported on the batch dimension."));
+ OP_REQUIRES(
+ context, ksize[3] == 1 && stride[3] == 1,
+ errors::Unimplemented(
+ "MaxPoolingGrad is not yet supported on the depth dimension."));
+
+ PoolParameters params{context, ksize, stride,
padding_, FORMAT_NHWC, tensor_in.shape()};
Tensor* output = nullptr;
OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
@@ -474,7 +563,7 @@ class MaxPoolingGradGradOp : public OpKernel {
// tensor_out_as_matrix with the corresponding values in
// top_diff_as_matrix.
auto shard = [&params, &in_mat, &out_mat, &top_diff_mat, &bottom_diff_mat](
- int64 start, int64 limit) {
+ int64 start, int64 limit) {
const int32 depth = params.depth;
const int32 in_rows = params.tensor_in_rows;
const int32 in_cols = params.tensor_in_cols;
@@ -555,20 +644,22 @@ class MaxPoolingGradGradOp<Eigen::GpuDevice, T> : public OpKernel {
OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format));
OP_REQUIRES(context, FormatFromString(data_format, &data_format_),
errors::InvalidArgument("Invalid data format"));
- OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
- OP_REQUIRES(context, ksize_.size() == 4,
- errors::InvalidArgument("Sliding window ksize field must "
- "specify 4 dimensions"));
- OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
- OP_REQUIRES(context, stride_.size() == 4,
- errors::InvalidArgument("Sliding window strides field must "
- "specify 4 dimensions"));
+ if (context->num_inputs() == 3) {
+ OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
+ OP_REQUIRES(context, ksize_.size() == 4,
+ errors::InvalidArgument("Sliding window ksize field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
+ OP_REQUIRES(context, stride_.size() == 4,
+ errors::InvalidArgument("Sliding window strides field must "
+ "specify 4 dimensions"));
+ const int32 ksize_n = GetTensorDim(ksize_, data_format_, 'N');
+ const int32 stride_n = GetTensorDim(stride_, data_format_, 'N');
+ OP_REQUIRES(context, ksize_n == 1 && stride_n == 1,
+ errors::Unimplemented(
+ "Pooling is not yet supported on the batch dimension."));
+ }
OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
- const int32 ksize_n = GetTensorDim(ksize_, data_format_, 'N');
- const int32 stride_n = GetTensorDim(stride_, data_format_, 'N');
- OP_REQUIRES(context, ksize_n == 1 && stride_n == 1,
- errors::Unimplemented(
- "Pooling is not yet supported on the batch dimension."));
}
void Compute(OpKernelContext* context) override {
@@ -590,7 +681,33 @@ class MaxPoolingGradGradOp<Eigen::GpuDevice, T> : public OpKernel {
OP_REQUIRES_OK(context,
context->allocate_output(0, tensor_out.shape(), &output));
- PoolParameters params{context, ksize_, stride_,
+ std::vector<int32> ksize = ksize_;
+ std::vector<int32> stride = stride_;
+ if (context->num_inputs() == 5) {
+ const Tensor& tensor_ksize = context->input(3);
+ auto value_ksize = tensor_ksize.flat<int32>();
+ ksize.resize(tensor_ksize.shape().num_elements());
+ std::copy_n(&value_ksize(0), ksize.size(), ksize.begin());
+
+ const Tensor& tensor_stride = context->input(4);
+ auto value_stride = tensor_stride.flat<int32>();
+ stride.resize(tensor_stride.shape().num_elements());
+ std::copy_n(&value_stride(0), stride.size(), stride.begin());
+ }
+
+ OP_REQUIRES(context, ksize.size() == 4,
+ errors::InvalidArgument("Sliding window ksize field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES(context, stride.size() == 4,
+ errors::InvalidArgument("Sliding window strides field must "
+ "specify 4 dimensions"));
+ const int32 ksize_n = GetTensorDim(ksize, data_format_, 'N');
+ const int32 stride_n = GetTensorDim(stride, data_format_, 'N');
+ OP_REQUIRES(context, ksize_n == 1 && stride_n == 1,
+ errors::Unimplemented(
+ "Pooling is not yet supported on the batch dimension."));
+
+ PoolParameters params{context, ksize, stride,
padding_, data_format_, tensor_in.shape()};
functor::MaxPoolGradBackwardNoMask<T>()(
@@ -670,6 +787,84 @@ class MaxPoolingNoMaskOp : public OpKernel {
};
template <typename Device, typename T>
+class MaxPoolingNoMaskV2Op : public OpKernel {
+ public:
+ explicit MaxPoolingNoMaskV2Op(OpKernelConstruction* context)
+ : OpKernel(context) {
+ string data_format;
+ OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format));
+ OP_REQUIRES(context, FormatFromString(data_format, &data_format_),
+ errors::InvalidArgument("Invalid data format"));
+ OP_REQUIRES(
+ context, data_format_ == FORMAT_NHWC,
+ errors::InvalidArgument(
+ "Default MaxPoolingNoMaskOp only supports NHWC on device type ",
+ DeviceTypeString(context->device_type())));
+ if (context->num_inputs() == 1) {
+ OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
+ OP_REQUIRES(context, ksize_.size() == 4,
+ errors::InvalidArgument("Sliding window ksize field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
+ OP_REQUIRES(context, stride_.size() == 4,
+ errors::InvalidArgument("Sliding window stride field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES(context, ksize_[0] == 1 && stride_[0] == 1,
+ errors::Unimplemented(
+ "Pooling is not yet supported on the batch dimension."));
+ }
+ OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+ }
+
+ void Compute(OpKernelContext* context) override {
+ const Tensor& tensor_in = context->input(0);
+
+ std::vector<int32> ksize = ksize_;
+ std::vector<int32> stride = stride_;
+
+ if (context->num_inputs() != 1) {
+ const Tensor& tensor_ksize = context->input(1);
+ auto value_ksize = tensor_ksize.flat<int32>();
+ ksize.resize(tensor_ksize.shape().num_elements());
+ std::copy_n(&value_ksize(0), ksize.size(), ksize.begin());
+
+ const Tensor& tensor_stride = context->input(2);
+ auto value_stride = tensor_stride.flat<int32>();
+ stride.resize(tensor_stride.shape().num_elements());
+ std::copy_n(&value_stride(0), stride.size(), stride.begin());
+ }
+ OP_REQUIRES(context, ksize.size() == 4,
+ errors::InvalidArgument("Sliding window ksize field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES(context, stride.size() == 4,
+ errors::InvalidArgument("Sliding window stride field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES(context, ksize[0] == 1 && stride[0] == 1,
+ errors::Unimplemented(
+ "Pooling is not yet supported on the batch dimension."));
+ PoolParameters params{context, ksize, stride,
+ padding_, data_format_, tensor_in.shape()};
+ if (!context->status().ok()) {
+ return;
+ }
+
+ TensorShape out_shape({params.tensor_in_batch, params.out_height,
+ params.out_width, params.depth});
+ Tensor* output = nullptr;
+ OP_REQUIRES_OK(context, context->allocate_output(0, out_shape, &output));
+
+ LaunchMaxPoolingNoMask<Device, T>::launch(context, params, tensor_in,
+ output);
+ }
+
+ private:
+ std::vector<int32> ksize_;
+ std::vector<int32> stride_;
+ Padding padding_;
+ TensorFormat data_format_;
+};
+
+template <typename Device, typename T>
struct LaunchMaxPoolingWithArgmax;
template <typename Device, typename T>
@@ -879,6 +1074,95 @@ class MaxPoolingNoMaskOp<GPUDevice, T> : public OpKernel {
};
template <typename T>
+class MaxPoolingNoMaskV2Op<GPUDevice, T> : public OpKernel {
+ public:
+ typedef GPUDevice Device;
+ explicit MaxPoolingNoMaskV2Op(OpKernelConstruction* context)
+ : OpKernel(context) {
+ string data_format;
+ OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format));
+ OP_REQUIRES(context, FormatFromString(data_format, &data_format_),
+ errors::InvalidArgument("Invalid data format"));
+ if (context->num_inputs() == 1) {
+ OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
+ OP_REQUIRES(context, ksize_.size() == 4,
+ errors::InvalidArgument("Sliding window ksize field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
+ OP_REQUIRES(context, stride_.size() == 4,
+ errors::InvalidArgument("Sliding window stride field must "
+ "specify 4 dimensions"));
+ const int32 ksize_n = GetTensorDim(ksize_, data_format_, 'N');
+ const int32 stride_n = GetTensorDim(stride_, data_format_, 'N');
+ OP_REQUIRES(context, ksize_n == 1 && stride_n == 1,
+ errors::Unimplemented(
+ "Pooling is not yet supported on the batch dimension."));
+ }
+ OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+ use_dnn_ = CanUseCudnn();
+ }
+
+ void Compute(OpKernelContext* context) override {
+ const Tensor& tensor_in = context->input(0);
+
+ std::vector<int32> ksize = ksize_;
+ std::vector<int32> stride = stride_;
+
+ if (context->num_inputs() != 1) {
+ const Tensor& tensor_ksize = context->input(1);
+ auto value_ksize = tensor_ksize.flat<int32>();
+ ksize.resize(tensor_ksize.shape().num_elements());
+ std::copy_n(&value_ksize(0), ksize.size(), ksize.begin());
+
+ const Tensor& tensor_stride = context->input(2);
+ auto value_stride = tensor_stride.flat<int32>();
+ stride.resize(tensor_stride.shape().num_elements());
+ std::copy_n(&value_stride(0), stride.size(), stride.begin());
+ }
+ OP_REQUIRES(context, ksize.size() == 4,
+ errors::InvalidArgument("Sliding window ksize field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES(context, stride.size() == 4,
+ errors::InvalidArgument("Sliding window stride field must "
+ "specify 4 dimensions"));
+ const int32 ksize_n = GetTensorDim(ksize, data_format_, 'N');
+ const int32 stride_n = GetTensorDim(stride, data_format_, 'N');
+ OP_REQUIRES(context, ksize_n == 1 && stride_n == 1,
+ errors::Unimplemented(
+ "Pooling is not yet supported on the batch dimension."));
+
+ PoolParameters params{context, ksize, stride,
+ padding_, data_format_, tensor_in.shape()};
+ if (!context->status().ok()) {
+ return;
+ }
+
+ TensorShape out_shape =
+ ShapeFromFormat(data_format_, params.tensor_in_batch, params.out_height,
+ params.out_width, params.depth);
+ if (use_dnn_ && data_format_ == FORMAT_NCHW) {
+ DnnPoolingOp<T>::Compute(
+ context, perftools::gputools::dnn::PoolingMode::kMaximum, ksize,
+ stride, padding_, data_format_, tensor_in, out_shape);
+ } else {
+ CHECK(data_format_ == FORMAT_NHWC)
+ << "Non-Cudnn MaxPool only supports NHWC format";
+ Tensor* output = nullptr;
+ OP_REQUIRES_OK(context, context->allocate_output(0, out_shape, &output));
+ LaunchMaxPoolingNoMask<Device, T>::launch(context, params, tensor_in,
+ output);
+ }
+ }
+
+ private:
+ std::vector<int32> ksize_;
+ std::vector<int32> stride_;
+ Padding padding_;
+ TensorFormat data_format_;
+ bool use_dnn_;
+};
+
+template <typename T>
struct LaunchMaxPoolingNoMask<Eigen::GpuDevice, T> {
static void launch(OpKernelContext* context, const PoolParameters& params,
const Tensor& input, Tensor* output) {
@@ -969,13 +1253,28 @@ struct LaunchMaxPoolingGradGradWithArgmax<Eigen::GpuDevice, T> {
MaxPoolingGradOp<D##Device, T>); \
REGISTER_KERNEL_BUILDER( \
Name("MaxPoolGradGrad").Device(DEVICE_##D).TypeConstraint<T>("T"), \
- MaxPoolingGradGradOp<D##Device, T>);
+ MaxPoolingGradGradOp<D##Device, T>); \
+ REGISTER_KERNEL_BUILDER(Name("MaxPoolGradV2") \
+ .Device(DEVICE_##D) \
+ .HostMemory("ksize") \
+ .HostMemory("strides") \
+ .TypeConstraint<T>("T"), \
+ MaxPoolingGradOp<D##Device, T>); \
+ REGISTER_KERNEL_BUILDER(Name("MaxPoolGradGradV2") \
+ .Device(DEVICE_##D) \
+ .HostMemory("ksize") \
+ .HostMemory("strides") \
+ .TypeConstraint<T>("T"), \
+ MaxPoolingGradGradOp<D##Device, T>);
// Below kernels implemented only for CPU device.
-#define REGISTER_CPU_ONLY_POOL_KERNELS(T) \
- REGISTER_KERNEL_BUILDER( \
- Name("MaxPool").Device(DEVICE_CPU).TypeConstraint<T>("T"), \
- MaxPoolingOp<CPUDevice, T>);
+#define REGISTER_CPU_ONLY_POOL_KERNELS(T) \
+ REGISTER_KERNEL_BUILDER( \
+ Name("MaxPool").Device(DEVICE_CPU).TypeConstraint<T>("T"), \
+ MaxPoolingOp<CPUDevice, T>); \
+ REGISTER_KERNEL_BUILDER( \
+ Name("MaxPoolV2").Device(DEVICE_CPU).TypeConstraint<T>("T"), \
+ MaxPoolingV2Op<CPUDevice, T>);
TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_ONLY_POOL_KERNELS);
#undef REGISTER_CPU_ONLY_POOL_KERNELS
@@ -1015,9 +1314,22 @@ TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_MAX_POOL_KERNELS);
.TypeConstraint<T>("T") \
.Label("eigen_tensor"), \
MaxPoolingOp<GPUDevice, T>); \
+ REGISTER_KERNEL_BUILDER(Name("MaxPoolV2") \
+ .Device(DEVICE_GPU) \
+ .HostMemory("ksize") \
+ .HostMemory("strides") \
+ .TypeConstraint<T>("T") \
+ .Label("eigen_tensor"), \
+ MaxPoolingV2Op<GPUDevice, T>); \
REGISTER_KERNEL_BUILDER( \
Name("MaxPool").Device(DEVICE_GPU).TypeConstraint<T>("T"), \
MaxPoolingNoMaskOp<GPUDevice, T>); \
+ REGISTER_KERNEL_BUILDER(Name("MaxPoolV2") \
+ .Device(DEVICE_GPU) \
+ .HostMemory("ksize") \
+ .HostMemory("strides") \
+ .TypeConstraint<T>("T"), \
+ MaxPoolingNoMaskV2Op<GPUDevice, T>); \
REGISTER_KERNEL_BUILDER(Name("MaxPoolWithArgmax") \
.Device(DEVICE_GPU) \
.TypeConstraint<int64>("Targmax") \
diff --git a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
index 3b23c72f0f..f81a448e51 100644
--- a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
@@ -206,15 +206,10 @@ class MklConv2DCustomBackpropFilterOp : public OpKernel {
// Mkl needs the entities in its native format.
// So create temporary tensors along with buffers to
// convert the received entities.
- Tensor mkl_tmp_input_buf_tensor, mkl_tmp_out_backprop_buf_tensor,
- mkl_tmp_buf_trans_input;
+ Tensor mkl_tmp_input_buf_tensor, mkl_tmp_out_backprop_buf_tensor;
// This preparation sets (1) dnnResourceSrc (2) dnnResourceDiffDst
- mkl_context.MklPrepareInputs(context, data_format_,
- input_in_mkl_format,
- out_backprop_in_mkl_format,
- &mkl_tmp_input_buf_tensor,
- &mkl_tmp_out_backprop_buf_tensor,
- &mkl_tmp_buf_trans_input);
+ mkl_context.MklPrepareInputs(context, &mkl_tmp_input_buf_tensor,
+ &mkl_tmp_out_backprop_buf_tensor);
// Final conv-grad-filter should be in TF layout.
Tensor* grad_filter;
@@ -312,58 +307,34 @@ class MklConv2DCustomBackpropFilterOp : public OpKernel {
// Compare incoming tensor layouts with MKL preferred layouts and convert
// data to the preferred layout if necessary
- void MklPrepareInputs(OpKernelContext* context, TensorFormat format,
- bool input_in_mkl_format,
- bool out_backprop_in_mkl_format,
+ void MklPrepareInputs(OpKernelContext* context,
Tensor* mkl_tmp_input_buf_tensor,
- Tensor* mkl_tmp_out_backprop_buf_tensor,
- Tensor* mkl_tmp_buf_trans_input) {
+ Tensor* mkl_tmp_out_backprop_buf_tensor) {
bool mkl_convert_input, mkl_convert_out_backprop;
dnnPrimitive_t mkl_prim_convert_input, mkl_prim_convert_out_backprop;
- dnnLayout_t mkl_lt_internal_input, mkl_lt_internal_out_backprop,
- mkl_lt_trans_input;
+ dnnLayout_t mkl_lt_internal_input, mkl_lt_internal_out_backprop;
void *mkl_buf_convert_input, *mkl_buf_convert_out_backprop;
- void *mkl_buf_input, *mkl_buf_out_backprop;
mkl_prim_convert_input = nullptr;
mkl_prim_convert_out_backprop = nullptr;
mkl_lt_internal_input = nullptr;
mkl_lt_internal_out_backprop = nullptr;
- mkl_lt_trans_input = nullptr;
mkl_buf_convert_input = nullptr;
mkl_buf_convert_out_backprop = nullptr;
- mkl_buf_input = nullptr;
- mkl_buf_out_backprop = nullptr;
// Compare with internal layouts and convert if needed
const Tensor& input = MklGetInput(context, 0);
- if (!input_in_mkl_format && format == FORMAT_NHWC){
- TensorShape nchw_shape = ShapeFromFormat(FORMAT_NCHW,
- in_sizes[MklDims::N], in_sizes[MklDims::H],
- in_sizes[MklDims::W], in_sizes[MklDims::C]);
- OP_REQUIRES_OK(context, context->allocate_temp(
- DataTypeToEnum<float>::value, nchw_shape, mkl_tmp_buf_trans_input));
- MklNHWCToNCHW(input, &mkl_tmp_buf_trans_input);
- mkl_buf_input = const_cast<void*>(static_cast<const void*>(
- mkl_tmp_buf_trans_input->flat<float>().data()));
- size_t strides[4];
- GetStridesFromSizes(FORMAT_NCHW, strides, in_sizes);
- CHECK_EQ(dnnLayoutCreate_F32(&mkl_lt_trans_input, in_dims, in_sizes,
- strides), E_SUCCESS);
- }
- else {
- mkl_buf_input =
- const_cast<void*>(static_cast<const void*>(input.flat<T>().data()));
- mkl_lt_trans_input = lt_input;
- }
+ void* mkl_buf_input =
+ const_cast<void*>(static_cast<const void*>(input.flat<T>().data()));
CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(
&mkl_lt_internal_input, prim_conv_bwdfilter, dnnResourceSrc),
E_SUCCESS);
mkl_convert_input =
- !dnnLayoutCompare_F32(mkl_lt_internal_input, mkl_lt_trans_input);
+ !dnnLayoutCompare_F32(mkl_lt_internal_input, lt_input);
if (mkl_convert_input) {
- CHECK_EQ(dnnConversionCreate_F32(&mkl_prim_convert_input,
- mkl_lt_trans_input, mkl_lt_internal_input), E_SUCCESS);
+ CHECK_EQ(dnnConversionCreate_F32(&mkl_prim_convert_input, lt_input,
+ mkl_lt_internal_input),
+ E_SUCCESS);
AllocTmpBuffer(context, mkl_tmp_input_buf_tensor, mkl_lt_internal_input,
&mkl_buf_convert_input);
CHECK_EQ(dnnConversionExecute_F32(mkl_prim_convert_input, mkl_buf_input,
@@ -372,30 +343,26 @@ class MklConv2DCustomBackpropFilterOp : public OpKernel {
dnnDelete_F32(mkl_prim_convert_input);
}
dnnLayoutDelete_F32(mkl_lt_internal_input);
- if (!input_in_mkl_format && format == FORMAT_NHWC)
- dnnLayoutDelete_F32(mkl_lt_trans_input);
-
conv_res[dnnResourceSrc] =
(mkl_convert_input) ? mkl_buf_convert_input : mkl_buf_input;
const Tensor& out_backprop = MklGetInput(context, 2);
- mkl_buf_out_backprop = const_cast<void*>(
- static_cast<const void*>(out_backprop.flat<T>().data()));
+ void* mkl_buf_out_backprop = const_cast<void*>(static_cast<const void*>(
+ out_backprop.flat<T>().data()));
CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(&mkl_lt_internal_out_backprop,
prim_conv_bwdfilter,
dnnResourceDiffDst),
E_SUCCESS);
mkl_convert_out_backprop =
- !dnnLayoutCompare_F32(mkl_lt_internal_out_backprop,
- lt_out_backprop);
+ !dnnLayoutCompare_F32(mkl_lt_internal_out_backprop, lt_out_backprop);
if (mkl_convert_out_backprop) {
CHECK_EQ(dnnConversionCreate_F32(&mkl_prim_convert_out_backprop,
lt_out_backprop, mkl_lt_internal_out_backprop),
E_SUCCESS);
AllocTmpBuffer(context, mkl_tmp_out_backprop_buf_tensor,
- mkl_lt_internal_out_backprop, &mkl_buf_convert_out_backprop);
+ lt_out_backprop, &mkl_buf_convert_out_backprop);
CHECK_EQ(dnnConversionExecute_F32(mkl_prim_convert_out_backprop,
mkl_buf_out_backprop,
mkl_buf_convert_out_backprop),
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 45d22556aa..203e694631 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -272,13 +272,11 @@ class MklConv2DOp : public OpKernel {
// Temp tensor used to allocate tmp buffers
Tensor mkl_tmp_input_buf_tensor, mkl_tmp_filter_buf_tensor,
- mkl_tmp_bias_buf_tensor, mkl_tmp_buf_trans_input;
- mkl_context.MklPrepareConvolutionInputs(context, data_format_,
- input_in_mkl_format,
+ mkl_tmp_bias_buf_tensor;
+ mkl_context.MklPrepareConvolutionInputs(context,
&mkl_tmp_input_buf_tensor,
&mkl_tmp_filter_buf_tensor,
- &mkl_tmp_bias_buf_tensor,
- &mkl_tmp_buf_trans_input);
+ &mkl_tmp_bias_buf_tensor);
// Execute convolution
CHECK_EQ(dnnExecute_F32(mkl_context.prim_fwd, mkl_context.conv_res),
@@ -329,59 +327,38 @@ class MklConv2DOp : public OpKernel {
// Compare incoming tensor layouts with MKL preferred layouts and convert
// data to the preferred layout if necessary
void MklPrepareConvolutionInputs(OpKernelContext* context,
- TensorFormat format,
- bool input_in_mkl_format,
Tensor* mkl_tmp_input_buf_tensor,
Tensor* mkl_tmp_filter_buf_tensor,
- Tensor* mkl_tmp_bias_buf_tensor,
- Tensor* mkl_tmp_buf_trans_input) {
+ Tensor* mkl_tmp_bias_buf_tensor) {
bool mkl_convert_input, mkl_convert_filter, mkl_convert_bias;
dnnPrimitive_t mkl_prim_convert_filter, mkl_prim_convert_bias,
mkl_prim_convert_input;
dnnLayout_t mkl_lt_internal_filter, mkl_lt_internal_bias,
- mkl_lt_internal_input, mkl_lt_trans_input;
+ mkl_lt_internal_input;
void *mkl_buf_convert_input, *mkl_buf_convert_filter,
- *mkl_buf_convert_bias, *mkl_buf_input;
+ *mkl_buf_convert_bias;
mkl_prim_convert_filter = nullptr;
mkl_prim_convert_bias = nullptr;
mkl_prim_convert_input = nullptr;
mkl_lt_internal_filter = nullptr;
mkl_lt_internal_bias = nullptr;
mkl_lt_internal_input = nullptr;
- mkl_lt_trans_input = nullptr;
mkl_buf_convert_input = nullptr;
mkl_buf_convert_filter = nullptr;
mkl_buf_convert_bias = nullptr;
- mkl_buf_input = nullptr;
// Compare with internal layouts and convert if needed
const Tensor& input = MklGetInput(context, 0);
- if (!input_in_mkl_format && format == FORMAT_NHWC) {
- TensorShape nchw_shape = ShapeFromFormat(FORMAT_NCHW,
- in_sizes[MklDims::N], in_sizes[MklDims::H],
- in_sizes[MklDims::W], in_sizes[MklDims::C]);
- OP_REQUIRES_OK(context, context->allocate_temp(
- DataTypeToEnum<float>::value, nchw_shape, mkl_tmp_buf_trans_input));
- MklNHWCToNCHW(input, &mkl_tmp_buf_trans_input);
- mkl_buf_input = const_cast<void*>(static_cast<const void*>(
- mkl_tmp_buf_trans_input->flat<float>().data()));
- size_t strides[4];
- GetStridesFromSizes(FORMAT_NCHW, strides, in_sizes);
- CHECK_EQ(dnnLayoutCreate_F32(&mkl_lt_trans_input, in_dims, in_sizes,
- strides), E_SUCCESS);
- } else {
- mkl_buf_input = const_cast<void*>(
- static_cast<const void*>(input.flat<T>().data()));
- mkl_lt_trans_input = lt_input;
- }
+ void* mkl_buf_input =
+ const_cast<void*>(static_cast<const void*>(input.flat<T>().data()));
CHECK_EQ(dnnLayoutCreateFromPrimitive_F32(&mkl_lt_internal_input,
prim_fwd, dnnResourceSrc),
E_SUCCESS);
mkl_convert_input =
- !dnnLayoutCompare_F32(mkl_lt_internal_input, mkl_lt_trans_input);
+ !dnnLayoutCompare_F32(mkl_lt_internal_input, lt_input);
if (mkl_convert_input) {
CHECK_EQ(dnnConversionCreate_F32(&mkl_prim_convert_input,
- mkl_lt_trans_input, mkl_lt_internal_input), E_SUCCESS);
+ lt_input, mkl_lt_internal_input), E_SUCCESS);
AllocTmpBuffer(context, mkl_tmp_input_buf_tensor, mkl_lt_internal_input,
&mkl_buf_convert_input);
CHECK_EQ(dnnConversionExecute_F32(mkl_prim_convert_input, mkl_buf_input,
@@ -390,8 +367,6 @@ class MklConv2DOp : public OpKernel {
dnnDelete_F32(mkl_prim_convert_input);
}
dnnLayoutDelete_F32(mkl_lt_internal_input);
- if (!input_in_mkl_format && format == FORMAT_NHWC)
- dnnLayoutDelete_F32(mkl_lt_trans_input);
conv_res[dnnResourceSrc] =
(mkl_convert_input) ? mkl_buf_convert_input : mkl_buf_input;
diff --git a/tensorflow/core/kernels/mkl_tfconv_op.cc b/tensorflow/core/kernels/mkl_tfconv_op.cc
index c8e5df32ce..b48c735d12 100644
--- a/tensorflow/core/kernels/mkl_tfconv_op.cc
+++ b/tensorflow/core/kernels/mkl_tfconv_op.cc
@@ -83,42 +83,16 @@ class MklToTfOp : public OpKernel {
OP_REQUIRES_OK(context,
context->allocate_output(0, output_shape, &output_tensor));
- // If data format is NHWC, transform MKL tensor to NCHW format and then
- // do NCHW -> NHWC.
- dnnLayout_t lt_trans_input = nullptr;
- Tensor mkl_tmp_trans_input_buf_tensor;
- void* buf_trans_input = nullptr;
- bool input_fmt_nhwc = input_shape.IsTensorInNHWCFormat();
- if (input_fmt_nhwc && ndims == 4 && has_avx512f_) {
- size_t strides_nchw[4];
- GetStridesFromSizes(FORMAT_NCHW, strides_nchw, in_sizes);
- CHECK_EQ(
- dnnLayoutCreate_F32(&lt_trans_input, ndims, in_sizes, strides_nchw),
- E_SUCCESS);
- AllocTmpBuffer(context, &mkl_tmp_trans_input_buf_tensor, lt_trans_input,
- &buf_trans_input);
- } else {
- lt_trans_input = static_cast<dnnLayout_t>(input_shape.GetTfLayout());
- buf_trans_input =
- static_cast<void*>(const_cast<T*>(output_tensor->flat<T>().data()));
- }
-
+ dnnLayout_t output_layout =
+ static_cast<dnnLayout_t>(input_shape.GetTfLayout());
// Execute DNNConversion.
void* input_buffer =
static_cast<void*>(const_cast<T*>(input_tensor.flat<T>().data()));
- input_shape.GetConvertedFlatData(lt_trans_input, input_buffer,
- buf_trans_input);
- // NCHW -> NHWC, if data format is NHWC
- if (input_fmt_nhwc && ndims == 4 && has_avx512f_) {
- dnnLayoutDelete_F32(lt_trans_input);
- TensorShape nhwc_shape = ShapeFromFormat(
- FORMAT_NHWC, in_sizes[MklDims::N], in_sizes[MklDims::H],
- in_sizes[MklDims::W], in_sizes[MklDims::C]);
- MklNCHWToNHWC(mkl_tmp_trans_input_buf_tensor, &output_tensor);
- }
-
delete[] in_sizes;
-
+ void* output_buffer =
+ static_cast<void*>(const_cast<T*>(output_tensor->flat<T>().data()));
+ input_shape.GetConvertedFlatData(output_layout, input_buffer,
+ output_buffer);
VLOG(1) << "MKLToTFConversion complete successfully.";
}
diff --git a/tensorflow/core/kernels/pack_op.cc b/tensorflow/core/kernels/pack_op.cc
index 75820e3106..814128d99a 100644
--- a/tensorflow/core/kernels/pack_op.cc
+++ b/tensorflow/core/kernels/pack_op.cc
@@ -158,6 +158,7 @@ REGISTER_PACK(string);
TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU);
TF_CALL_int64(REGISTER_GPU);
+REGISTER_GPU(bool);
#undef REGISTER_GPU
// A special GPU kernel for int32.
diff --git a/tensorflow/core/kernels/pooling_ops_common.h b/tensorflow/core/kernels/pooling_ops_common.h
index 2c097c0ce2..1b59c18df7 100644
--- a/tensorflow/core/kernels/pooling_ops_common.h
+++ b/tensorflow/core/kernels/pooling_ops_common.h
@@ -69,6 +69,8 @@ struct PoolParameters {
};
// An implementation of MaxPooling (forward).
+// TODO (yongtang): Remove MaxPoolingOp and use MaxPoolingV2Op,
+// QuantizedMaxPoolingOp depends on MaxPoolingOp so keep intact for now
template <typename Device, typename T>
class MaxPoolingOp : public OpKernel {
public:
@@ -255,6 +257,219 @@ class MaxPoolingOp : public OpKernel {
};
template <typename Device, typename T>
+class MaxPoolingV2Op : public OpKernel {
+ public:
+ explicit MaxPoolingV2Op(OpKernelConstruction* context) : OpKernel(context) {
+ string data_format;
+ auto status = context->GetAttr("data_format", &data_format);
+ if (status.ok()) {
+ OP_REQUIRES(context, FormatFromString(data_format, &data_format_),
+ errors::InvalidArgument("Invalid data format"));
+ OP_REQUIRES(
+ context, data_format_ == FORMAT_NHWC,
+ errors::InvalidArgument("Default MaxPoolingOp only supports NHWC."));
+ } else {
+ data_format_ = FORMAT_NHWC;
+ }
+ if (context->num_inputs() == 1) {
+ OP_REQUIRES_OK(context, context->GetAttr("ksize", &ksize_));
+ OP_REQUIRES(context, ksize_.size() == 4,
+ errors::InvalidArgument("Sliding window ksize field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES_OK(context, context->GetAttr("strides", &stride_));
+ OP_REQUIRES(context, stride_.size() == 4,
+ errors::InvalidArgument("Sliding window stride field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES(context, ksize_[0] == 1 && stride_[0] == 1,
+ errors::Unimplemented(
+ "Pooling is not yet supported on the batch dimension."));
+ }
+ OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+ }
+
+ void Compute(OpKernelContext* context) override {
+ const Tensor& tensor_in = context->input(0);
+
+ std::vector<int32> ksize = ksize_;
+ std::vector<int32> stride = stride_;
+
+ if (context->num_inputs() != 1) {
+ const Tensor& tensor_ksize = context->input(1);
+ auto value_ksize = tensor_ksize.flat<int32>();
+ ksize.resize(tensor_ksize.shape().num_elements());
+ std::copy_n(&value_ksize(0), ksize.size(), ksize.begin());
+
+ const Tensor& tensor_stride = context->input(2);
+ auto value_stride = tensor_stride.flat<int32>();
+ stride.resize(tensor_stride.shape().num_elements());
+ std::copy_n(&value_stride(0), stride.size(), stride.begin());
+ }
+
+ OP_REQUIRES(context, ksize.size() == 4,
+ errors::InvalidArgument("Sliding window ksize field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES(context, stride.size() == 4,
+ errors::InvalidArgument("Sliding window stride field must "
+ "specify 4 dimensions"));
+ OP_REQUIRES(context, ksize[0] == 1 && stride[0] == 1,
+ errors::Unimplemented(
+ "Pooling is not yet supported on the batch dimension."));
+
+ PoolParameters params{context, ksize, stride,
+ padding_, FORMAT_NHWC, tensor_in.shape()};
+ if (!context->status().ok()) {
+ return;
+ }
+
+ Tensor* output = nullptr;
+ OP_REQUIRES_OK(context, context->allocate_output(
+ 0, params.forward_output_shape(), &output));
+
+ if (params.depth_window > 1) {
+ // Validate spec against the current implementation. A
+ // relaxation of these requirements would be ideal.
+ OP_REQUIRES(context, params.depth % params.depth_window == 0,
+ errors::Unimplemented(
+ "Depthwise max pooling requires "
+ "the depth window to evenly divide the input depth."));
+ OP_REQUIRES(
+ context, params.depth_window == params.depth_stride,
+ errors::Unimplemented("Depthwise max pooling requires "
+ "the depth window to equal the depth stride."));
+
+ DepthwiseMaxPool(context, output, tensor_in, params);
+ } else {
+ SpatialMaxPool(context, output, tensor_in, params, padding_);
+ }
+ }
+
+ private:
+ // Single-threaded implementation of DepthwiseMaxPool which
+ // does not handle all of the same options as SpatialMaxPool
+ // (strict assumptions on no padding, stride).
+ //
+ // TODO(vrv): implement a more general depthwise-max pool that works
+ // on GPU as well.
+ void DepthwiseMaxPool(OpKernelContext* context, Tensor* output,
+ const Tensor& tensor_in, const PoolParameters& params) {
+ Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic>>
+ in_by_pool(tensor_in.flat<T>().data(), params.depth_window,
+ tensor_in.NumElements() / params.depth_window);
+ Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic>> out_by_pool(
+ output->flat<T>().data(), 1, output->NumElements());
+ out_by_pool = in_by_pool.colwise().maxCoeff();
+ }
+
+ void SpatialMaxPool(OpKernelContext* context, Tensor* output,
+ const Tensor& tensor_in, const PoolParameters& params,
+ const Padding& padding) {
+ // On GPU, use Eigen's Spatial Max Pooling. On CPU, use an
+ // EigenMatrix version that is currently faster than Eigen's
+ // Spatial MaxPooling implementation.
+ //
+ // TODO(vrv): Remove this once we no longer need it.
+ if (std::is_same<Device, GPUDevice>::value) {
+ Eigen::PaddingType pt = BrainPadding2EigenPadding(padding);
+ functor::SpatialMaxPooling<Device, T>()(
+ context->eigen_device<Device>(), output->tensor<T, 4>(),
+ tensor_in.tensor<T, 4>(), params.window_rows, params.window_cols,
+ params.row_stride, params.col_stride, pt);
+ } else {
+ typedef Eigen::Map<const Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic>>
+ ConstEigenMatrixMap;
+ typedef Eigen::Map<Eigen::Matrix<T, Eigen::Dynamic, Eigen::Dynamic>>
+ EigenMatrixMap;
+
+ ConstEigenMatrixMap in_mat(tensor_in.flat<T>().data(), params.depth,
+ params.tensor_in_cols * params.tensor_in_rows *
+ params.tensor_in_batch);
+ EigenMatrixMap out_mat(
+ output->flat<T>().data(), params.depth,
+ params.out_width * params.out_height * params.tensor_in_batch);
+
+ const DeviceBase::CpuWorkerThreads& worker_threads =
+ *(context->device()->tensorflow_cpu_worker_threads());
+
+ // The following code basically does the following:
+ // 1. Flattens the input and output tensors into two dimensional arrays.
+ // tensor_in_as_matrix:
+ // depth by (tensor_in_cols * tensor_in_rows * tensor_in_batch)
+ // output_as_matrix:
+ // depth by (out_width * out_height * tensor_in_batch)
+ //
+ // 2. Walks through the set of columns in the flattened
+ // tensor_in_as_matrix,
+ // and updates the corresponding column(s) in output_as_matrix with the
+ // max value.
+ auto shard = [&params, &in_mat, &out_mat](int64 start, int64 limit) {
+
+ const int32 in_rows = params.tensor_in_rows;
+ const int32 in_cols = params.tensor_in_cols;
+ const int32 pad_rows = params.pad_rows;
+ const int32 pad_cols = params.pad_cols;
+ const int32 window_rows = params.window_rows;
+ const int32 window_cols = params.window_cols;
+ const int32 row_stride = params.row_stride;
+ const int32 col_stride = params.col_stride;
+ const int32 out_height = params.out_height;
+ const int32 out_width = params.out_width;
+
+ {
+ // Initializes the output tensor with MIN<T>.
+ const int32 output_image_size = out_height * out_width * params.depth;
+ EigenMatrixMap out_shard(out_mat.data() + start * output_image_size,
+ 1, (limit - start) * output_image_size);
+ out_shard.setConstant(Eigen::NumTraits<T>::lowest());
+ }
+
+ for (int32 b = start; b < limit; ++b) {
+ const int32 out_offset_batch = b * out_height;
+ for (int32 h = 0; h < in_rows; ++h) {
+ for (int32 w = 0; w < in_cols; ++w) {
+ // (h_start, h_end) * (w_start, w_end) is the range that the input
+ // vector projects to.
+ const int32 hpad = h + pad_rows;
+ const int32 wpad = w + pad_cols;
+ const int32 h_start = (hpad < window_rows)
+ ? 0
+ : (hpad - window_rows) / row_stride + 1;
+ const int32 h_end = std::min(hpad / row_stride + 1, out_height);
+ const int32 w_start = (wpad < window_cols)
+ ? 0
+ : (wpad - window_cols) / col_stride + 1;
+ const int32 w_end = std::min(wpad / col_stride + 1, out_width);
+ // compute elementwise max
+ const int32 in_offset = (b * in_rows + h) * in_cols + w;
+ for (int32 ph = h_start; ph < h_end; ++ph) {
+ const int32 out_offset_base =
+ (out_offset_batch + ph) * out_width;
+ for (int32 pw = w_start; pw < w_end; ++pw) {
+ const int32 out_offset = out_offset_base + pw;
+ out_mat.col(out_offset) =
+ out_mat.col(out_offset).cwiseMax(in_mat.col(in_offset));
+ }
+ }
+ }
+ }
+ }
+ };
+
+ // TODO(andydavis) Consider sharding across batch x rows x cols.
+ // TODO(andydavis) Consider a higher resolution shard cost model.
+ const int64 shard_cost =
+ params.tensor_in_rows * params.tensor_in_cols * params.depth;
+ Shard(worker_threads.num_threads, worker_threads.workers,
+ params.tensor_in_batch, shard_cost, shard);
+ }
+ }
+
+ std::vector<int32> ksize_;
+ std::vector<int32> stride_;
+ Padding padding_;
+ TensorFormat data_format_;
+};
+
+template <typename Device, typename T>
void SpatialAvgPool(OpKernelContext* context, Tensor* output,
const Tensor& input, const PoolParameters& params,
const Padding& padding) {
diff --git a/tensorflow/core/kernels/qr_op_impl.h b/tensorflow/core/kernels/qr_op_impl.h
index 029ef83480..ab664fa6d3 100644
--- a/tensorflow/core/kernels/qr_op_impl.h
+++ b/tensorflow/core/kernels/qr_op_impl.h
@@ -20,6 +20,10 @@ limitations under the License.
// improve compilation times.
#include <algorithm>
+#ifdef INTEL_MKL
+#define EIGEN_USE_MKL_ALL
+#endif // INTEL_MKL
+
#include "third_party/eigen3/Eigen/QR"
#include "tensorflow/core/framework/kernel_def_builder.h"
#include "tensorflow/core/framework/op_kernel.h"
diff --git a/tensorflow/core/kernels/reshape_op.cc b/tensorflow/core/kernels/reshape_op.cc
index 04454b76c1..16db8a6bb1 100644
--- a/tensorflow/core/kernels/reshape_op.cc
+++ b/tensorflow/core/kernels/reshape_op.cc
@@ -32,6 +32,7 @@ REGISTER_KERNEL_BUILDER(Name("Reshape")
.TypeConstraint<int32>("Tshape"), \
ReshapeOp);
TF_CALL_NUMBER_TYPES_NO_INT32(REGISTER_GPU_KERNEL);
+REGISTER_GPU_KERNEL(bool);
#undef REGISTER_GPU_KERNEL
#ifdef TENSORFLOW_USE_SYCL
diff --git a/tensorflow/core/kernels/sparse_matmul_op.h b/tensorflow/core/kernels/sparse_matmul_op.h
index 098b2d6500..308b641b54 100644
--- a/tensorflow/core/kernels/sparse_matmul_op.h
+++ b/tensorflow/core/kernels/sparse_matmul_op.h
@@ -153,6 +153,32 @@ EIGEN_STRONG_INLINE Packet4f pload2bf16<Packet4f>(const float* from) {
}
#endif
+#if defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX)
+// Return a packet with the first value of the input Packet replicated
+template <>
+EIGEN_STRONG_INLINE Packet4f pbroadcast_first<Packet4f>(const Packet4f& a) {
+ return vec_splat (a, 0);
+}
+
+// Return a packet with the second value of the input Packet replicated
+template <>
+EIGEN_STRONG_INLINE Packet4f pbroadcast_second<Packet4f>(const Packet4f& a) {
+ return vec_splat (a, 1);
+}
+
+// Return a packet with the third value of the input Packet replicated
+template <>
+EIGEN_STRONG_INLINE Packet4f pbroadcast_third<Packet4f>(const Packet4f& a) {
+ return vec_splat (a, 2);
+}
+
+// Return a packet with the fourth value of the input Packet replicated
+template <>
+EIGEN_STRONG_INLINE Packet4f pbroadcast_fourth<Packet4f>(const Packet4f& a) {
+ return vec_splat (a, 3);
+}
+#endif
+
#ifdef EIGEN_VECTORIZE_SSE2
// For PacketSize of 4 floats the Packet is not modified
template <>
diff --git a/tensorflow/core/kernels/tile_ops.cc b/tensorflow/core/kernels/tile_ops.cc
index f1da3c8afb..c49ebc0685 100644
--- a/tensorflow/core/kernels/tile_ops.cc
+++ b/tensorflow/core/kernels/tile_ops.cc
@@ -538,6 +538,12 @@ REGISTER_KERNEL_BUILDER(Name("Tile")
TileOp<GPUDevice>);
REGISTER_KERNEL_BUILDER(Name("Tile")
.Device(DEVICE_GPU)
+ .TypeConstraint<int32>("T")
+ .TypeConstraint<int32>("Tmultiples")
+ .HostMemory("multiples"),
+ TileOp<GPUDevice>);
+REGISTER_KERNEL_BUILDER(Name("Tile")
+ .Device(DEVICE_GPU)
.TypeConstraint<complex64>("T")
.TypeConstraint<int32>("Tmultiples")
.HostMemory("multiples"),
@@ -575,6 +581,12 @@ REGISTER_KERNEL_BUILDER(Name("TileGrad")
TileGradientOp<GPUDevice>);
REGISTER_KERNEL_BUILDER(Name("TileGrad")
.Device(DEVICE_GPU)
+ .TypeConstraint<int32>("T")
+ .TypeConstraint<int32>("Tmultiples")
+ .HostMemory("multiples"),
+ TileGradientOp<GPUDevice>);
+REGISTER_KERNEL_BUILDER(Name("TileGrad")
+ .Device(DEVICE_GPU)
.TypeConstraint<complex64>("T")
.TypeConstraint<int32>("Tmultiples")
.HostMemory("multiples"),
diff --git a/tensorflow/core/kernels/unique_op.cc b/tensorflow/core/kernels/unique_op.cc
index 6e51696d6f..701c5f6d2b 100644
--- a/tensorflow/core/kernels/unique_op.cc
+++ b/tensorflow/core/kernels/unique_op.cc
@@ -26,7 +26,7 @@ namespace tensorflow {
typedef Eigen::ThreadPoolDevice CPUDevice;
-template <typename T>
+template <typename T, typename TIndex>
class UniqueOp : public OpKernel {
public:
explicit UniqueOp(OpKernelConstruction* context) : OpKernel(context) {}
@@ -48,9 +48,9 @@ class UniqueOp : public OpKernel {
Tensor* idx = nullptr;
OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
{0}, 1, input.shape(), &idx));
- auto idx_vec = idx->template vec<int32>();
+ auto idx_vec = idx->template vec<TIndex>();
- std::unordered_map<T, int32> uniq;
+ std::unordered_map<T, TIndex> uniq;
uniq.reserve(2 * N);
for (int64 i = 0, j = 0; i < N; ++i) {
auto it = uniq.insert(std::make_pair(Tin(i), j));
@@ -72,7 +72,7 @@ class UniqueOp : public OpKernel {
if (num_outputs() > 2) {
OP_REQUIRES_OK(context, context->allocate_output(
2, TensorShape({uniq_size}), &output));
- auto count_output_vec = output->template vec<int32>();
+ auto count_output_vec = output->template vec<TIndex>();
count_output_vec.setZero();
for (int64 i = 0; i < N; ++i) {
count_output_vec(idx_vec(i))++;
@@ -86,12 +86,22 @@ class UniqueOp : public OpKernel {
.Device(DEVICE_CPU) \
.TypeConstraint<type>("T") \
.TypeConstraint<int32>("out_idx"), \
- UniqueOp<type>); \
+ UniqueOp<type, int32>); \
+ REGISTER_KERNEL_BUILDER(Name("Unique") \
+ .Device(DEVICE_CPU) \
+ .TypeConstraint<type>("T") \
+ .TypeConstraint<int64>("out_idx"), \
+ UniqueOp<type, int64>); \
REGISTER_KERNEL_BUILDER(Name("UniqueWithCounts") \
.Device(DEVICE_CPU) \
.TypeConstraint<type>("T") \
.TypeConstraint<int32>("out_idx"), \
- UniqueOp<type>)
+ UniqueOp<type, int32>) \
+ REGISTER_KERNEL_BUILDER(Name("UniqueWithCounts") \
+ .Device(DEVICE_CPU) \
+ .TypeConstraint<type>("T") \
+ .TypeConstraint<int64>("out_idx"), \
+ UniqueOp<type, int64>)
TF_CALL_REAL_NUMBER_TYPES(REGISTER_UNIQUE);
REGISTER_UNIQUE(string)
#undef REGISTER_UNIQUE
@@ -107,7 +117,15 @@ REGISTER_KERNEL_BUILDER(Name("Unique")
.HostMemory("x")
.HostMemory("y")
.HostMemory("idx"),
- UniqueOp<int32>);
+ UniqueOp<int32, int32>);
+REGISTER_KERNEL_BUILDER(Name("Unique")
+ .Device(DEVICE_GPU)
+ .TypeConstraint<int32>("T")
+ .TypeConstraint<int64>("out_idx")
+ .HostMemory("x")
+ .HostMemory("y")
+ .HostMemory("idx"),
+ UniqueOp<int32, int64>);
REGISTER_KERNEL_BUILDER(Name("Unique")
.Device(DEVICE_GPU)
.TypeConstraint<int64>("T")
@@ -115,7 +133,15 @@ REGISTER_KERNEL_BUILDER(Name("Unique")
.HostMemory("x")
.HostMemory("y")
.HostMemory("idx"),
- UniqueOp<int64>);
+ UniqueOp<int64, int32>);
+REGISTER_KERNEL_BUILDER(Name("Unique")
+ .Device(DEVICE_GPU)
+ .TypeConstraint<int64>("T")
+ .TypeConstraint<int64>("out_idx")
+ .HostMemory("x")
+ .HostMemory("y")
+ .HostMemory("idx"),
+ UniqueOp<int64, int64>);
#ifdef TENSORFLOW_USE_SYCL
REGISTER_KERNEL_BUILDER(Name("Unique")
@@ -125,7 +151,7 @@ REGISTER_KERNEL_BUILDER(Name("Unique")
.HostMemory("x")
.HostMemory("y")
.HostMemory("idx"),
- UniqueOp<int32>);
+ UniqueOp<int32, int32>);
REGISTER_KERNEL_BUILDER(Name("Unique")
.Device(DEVICE_SYCL)
.TypeConstraint<int64>("T")
@@ -133,6 +159,22 @@ REGISTER_KERNEL_BUILDER(Name("Unique")
.HostMemory("x")
.HostMemory("y")
.HostMemory("idx"),
- UniqueOp<int64>);
+ UniqueOp<int64, int32>);
+REGISTER_KERNEL_BUILDER(Name("Unique")
+ .Device(DEVICE_SYCL)
+ .TypeConstraint<int32>("T")
+ .TypeConstraint<int64>("out_idx")
+ .HostMemory("x")
+ .HostMemory("y")
+ .HostMemory("idx"),
+ UniqueOp<int32, int64>);
+REGISTER_KERNEL_BUILDER(Name("Unique")
+ .Device(DEVICE_SYCL)
+ .TypeConstraint<int64>("T")
+ .TypeConstraint<int64>("out_idx")
+ .HostMemory("x")
+ .HostMemory("y")
+ .HostMemory("idx"),
+ UniqueOp<int64, int64>);
#endif // TENSORFLOW_USE_SYCL
} // namespace tensorflow
diff --git a/tensorflow/core/kernels/unpack_op.cc b/tensorflow/core/kernels/unpack_op.cc
index c3bebfcbf9..7fd1def1fe 100644
--- a/tensorflow/core/kernels/unpack_op.cc
+++ b/tensorflow/core/kernels/unpack_op.cc
@@ -69,6 +69,8 @@ class UnpackOp : public OpKernel {
std::numeric_limits<Eigen::DenseIndex>::max()),
errors::InvalidArgument("output size must fit in Eigen DenseIndex"));
+// This optimization is currently not applicable for SYCL devices
+#ifndef TENSORFLOW_USE_SYCL
// Special case: Aligned, so we can share the underlying buffer.
//
// Apply this optimization conservatively: if input is aligned,
@@ -85,6 +87,7 @@ class UnpackOp : public OpKernel {
}
return;
}
+#endif // TENSORFLOW_USE_SYCL
int64 before_dim = 1;
for (int i = 0; i < axis; ++i) {
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-29 08:54:59 +0000