diff options
Diffstat (limited to 'unsupported/Eigen/CXX11')
55 files changed, 2377 insertions, 1519 deletions
diff --git a/unsupported/Eigen/CXX11/Core b/unsupported/Eigen/CXX11/Core index c8dcf7c16..946145f5a 100644 --- a/unsupported/Eigen/CXX11/Core +++ b/unsupported/Eigen/CXX11/Core @@ -33,13 +33,16 @@ #include <vector> #include "src/Core/util/EmulateArray.h" +#include "src/Core/util/MaxSizeVector.h" // Emulate the cxx11 functionality that we need if the compiler doesn't support it. -#if __cplusplus <= 199711L -#include "src/Core/util/EmulateCXX11Meta.h" -#else +// Visual studio 2015 doesn't advertise itself as cxx11 compliant, although it +// supports enough of the standard for our needs +#if __cplusplus > 199711L || EIGEN_COMP_MSVC >= 1900 #include "src/Core/util/CXX11Workarounds.h" #include "src/Core/util/CXX11Meta.h" +#else +#include "src/Core/util/EmulateCXX11Meta.h" #endif #include <Eigen/src/Core/util/ReenableStupidWarnings.h> diff --git a/unsupported/Eigen/CXX11/Tensor b/unsupported/Eigen/CXX11/Tensor index c681d3c20..16132398d 100644 --- a/unsupported/Eigen/CXX11/Tensor +++ b/unsupported/Eigen/CXX11/Tensor @@ -28,14 +28,22 @@ #include <cstddef> #include <cstring> + +#ifdef _WIN32 +typedef __int32 int32_t; +typedef unsigned __int32 uint32_t; +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; +#else #include <stdint.h> +#endif -#if __cplusplus > 199711 +#if __cplusplus > 199711 || EIGEN_COMP_MSVC >= 1900 #include <random> #endif #ifdef _WIN32 -#include <winbase.h> +#include <windows.h> #elif defined(__APPLE__) #include <mach/mach_time.h> #else @@ -43,6 +51,7 @@ #endif #ifdef EIGEN_USE_THREADS +#include <atomic> #include <condition_variable> #include <deque> #include <mutex> @@ -50,6 +59,7 @@ #endif #ifdef EIGEN_USE_GPU +#include <iostream> #include <cuda_runtime.h> #if defined(__CUDACC__) #include <curand_kernel.h> @@ -80,6 +90,8 @@ #include "src/Tensor/TensorReductionCuda.h" #include "src/Tensor/TensorArgMax.h" #include "src/Tensor/TensorConcatenation.h" +#include "src/Tensor/TensorContractionMapper.h" +#include "src/Tensor/TensorContractionBlocking.h" #include "src/Tensor/TensorContraction.h" #include "src/Tensor/TensorContractionThreadPool.h" #include "src/Tensor/TensorContractionCuda.h" diff --git a/unsupported/Eigen/CXX11/src/Core/util/CXX11Meta.h b/unsupported/Eigen/CXX11/src/Core/util/CXX11Meta.h index 3f149c6a3..c582e21f5 100644 --- a/unsupported/Eigen/CXX11/src/Core/util/CXX11Meta.h +++ b/unsupported/Eigen/CXX11/src/Core/util/CXX11Meta.h @@ -109,11 +109,9 @@ template<int n, typename x> struct get; template<int n, typename a, typename... as> struct get<n, type_list<a, as...>> : get<n-1, type_list<as...>> {}; template<typename a, typename... as> struct get<0, type_list<a, as...>> { typedef a type; }; -template<int n EIGEN_TPL_PP_SPEC_HACK_DEFC(typename, as)> struct get<n, type_list<EIGEN_TPL_PP_SPEC_HACK_USE(as)>> { static_assert((n - n) < 0, "meta-template get: The element to extract from a list must be smaller than the size of the list."); }; template<typename T, int n, T a, T... as> struct get<n, numeric_list<T, a, as...>> : get<n-1, numeric_list<T, as...>> {}; template<typename T, T a, T... as> struct get<0, numeric_list<T, a, as...>> { constexpr static T value = a; }; -template<typename T, int n EIGEN_TPL_PP_SPEC_HACK_DEFC(T, as)> struct get<n, numeric_list<T EIGEN_TPL_PP_SPEC_HACK_USEC(as)>> { static_assert((n - n) < 0, "meta-template get: The element to extract from a list must be smaller than the size of the list."); }; /* always get type, regardless of dummy; good for parameter pack expansion */ @@ -261,22 +259,20 @@ template< template< typename Reducer, - typename A, - typename... Ts -> struct reduce<Reducer, A, Ts...> + typename A +> struct reduce<Reducer, A> { - constexpr static inline A run(A a, Ts...) { return a; } + constexpr static inline A run(A a) { return a; } }; template< typename Reducer, typename A, - typename B, typename... Ts -> struct reduce<Reducer, A, B, Ts...> +> struct reduce<Reducer, A, Ts...> { - constexpr static inline auto run(A a, B b, Ts... ts) -> decltype(Reducer::run(a, reduce<Reducer, B, Ts...>::run(b, ts...))) { - return Reducer::run(a, reduce<Reducer, B, Ts...>::run(b, ts...)); + constexpr static inline auto run(A a, Ts... ts) -> decltype(Reducer::run(a, reduce<Reducer, Ts...>::run(ts...))) { + return Reducer::run(a, reduce<Reducer, Ts...>::run(ts...)); } }; diff --git a/unsupported/Eigen/CXX11/src/Core/util/CXX11Workarounds.h b/unsupported/Eigen/CXX11/src/Core/util/CXX11Workarounds.h index b1528aa66..fe4d22803 100644 --- a/unsupported/Eigen/CXX11/src/Core/util/CXX11Workarounds.h +++ b/unsupported/Eigen/CXX11/src/Core/util/CXX11Workarounds.h @@ -29,8 +29,10 @@ /* Check that the compiler at least claims to support C++11. It might not be sufficient * because the compiler may not implement it correctly, but at least we'll know. + * On the other hand, visual studio still doesn't claim to support C++11 although it's + * compliant enugh for our purpose. */ -#if __cplusplus <= 199711L +#if (__cplusplus <= 199711L) && (EIGEN_COMP_MSVC < 1900) #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) #pragma GCC diagnostic error "-Wfatal-errors" #endif diff --git a/unsupported/Eigen/CXX11/src/Core/util/EmulateArray.h b/unsupported/Eigen/CXX11/src/Core/util/EmulateArray.h index ab9c2ec3e..579519b04 100644 --- a/unsupported/Eigen/CXX11/src/Core/util/EmulateArray.h +++ b/unsupported/Eigen/CXX11/src/Core/util/EmulateArray.h @@ -13,9 +13,9 @@ // The array class is only available starting with cxx11. Emulate our own here -// if needed. +// if needed. Beware, msvc still doesn't advertise itself as a c++11 compiler! // Moreover, CUDA doesn't support the STL containers, so we use our own instead. -#if __cplusplus <= 199711L || defined(__CUDACC__) || defined(EIGEN_AVOID_STL_ARRAY) +#if (__cplusplus <= 199711L && EIGEN_COMP_MSVC < 1900) || defined(__CUDACC__) || defined(EIGEN_AVOID_STL_ARRAY) namespace Eigen { template <typename T, size_t n> class array { @@ -25,6 +25,16 @@ template <typename T, size_t n> class array { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& operator[] (size_t index) const { return values[index]; } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE T& front() { return values[0]; } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const T& front() const { return values[0]; } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE T& back() { return values[n-1]; } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const T& back() const { return values[n-1]; } + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static std::size_t size() { return n; } @@ -32,7 +42,7 @@ template <typename T, size_t n> class array { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array() { } - explicit EIGEN_DEVICE_FUNC + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array(const T& v) { EIGEN_STATIC_ASSERT(n==1, YOU_MADE_A_PROGRAMMING_MISTAKE) values[0] = v; @@ -123,27 +133,63 @@ template <typename T> class array<T, 0> { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& operator[] (size_t) { eigen_assert(false && "Can't index a zero size array"); - return *static_cast<T*>(NULL); + return dummy; } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& operator[] (size_t) const { eigen_assert(false && "Can't index a zero size array"); - return *static_cast<const T*>(NULL); + return dummy; } - static EIGEN_ALWAYS_INLINE std::size_t size() { return 0; } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE T& front() { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const T& front() const { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE T& back() { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const T& back() const { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::size_t size() { return 0; } EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE array() { } + EIGEN_STRONG_INLINE array() : dummy() { } #ifdef EIGEN_HAS_VARIADIC_TEMPLATES - array(std::initializer_list<T> l) { + EIGEN_DEVICE_FUNC array(std::initializer_list<T> l) : dummy() { eigen_assert(l.size() == 0); } #endif + + private: + T dummy; }; +// Comparison operator +// Todo: implement !=, <, <=, >, and >= +template<class T, std::size_t N> +EIGEN_DEVICE_FUNC bool operator==(const array<T,N>& lhs, const array<T,N>& rhs) { + for (std::size_t i = 0; i < N; ++i) { + if (lhs[i] != rhs[i]) { + return false; + } + } + return true; +} + + namespace internal { template<std::size_t I, class T, std::size_t N> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& array_get(array<T,N>& a) { diff --git a/unsupported/Eigen/CXX11/src/Core/util/MaxSizeVector.h b/unsupported/Eigen/CXX11/src/Core/util/MaxSizeVector.h new file mode 100644 index 000000000..551124bae --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Core/util/MaxSizeVector.h @@ -0,0 +1,130 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_FIXEDSIZEVECTOR_H +#define EIGEN_FIXEDSIZEVECTOR_H + +namespace Eigen { + +/** \class MaxSizeVector + * \ingroup Core + * + * \brief The MaxSizeVector class. + * + * The %MaxSizeVector provides a subset of std::vector functionality. + * + * The goal is to provide basic std::vector operations when using + * std::vector is not an option (e.g. on GPU or when compiling using + * FMA/AVX, as this can cause either compilation failures or illegal + * instruction failures). + * + * Beware: The constructors are not API compatible with these of + * std::vector. + */ +template <typename T> +class MaxSizeVector { + public: + // Construct a new MaxSizeVector, reserve n elements. + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit MaxSizeVector(size_t n) + : reserve_(n), size_(0), + data_(static_cast<T*>(internal::aligned_malloc(n * sizeof(T)))) { + for (size_t i = 0; i < n; ++i) { new (&data_[i]) T; } + } + + // Construct a new MaxSizeVector, reserve and resize to n. + // Copy the init value to all elements. + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + explicit MaxSizeVector(size_t n, const T& init) + : reserve_(n), size_(n), + data_(static_cast<T*>(internal::aligned_malloc(n * sizeof(T)))) { + for (size_t i = 0; i < n; ++i) { new (&data_[i]) T(init); } + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + ~MaxSizeVector() { + for (size_t i = 0; i < size_; ++i) { + data_[i].~T(); + } + internal::aligned_free(data_); + } + + // Append new elements (up to reserved size). + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void push_back(const T& t) { + eigen_assert(size_ < reserve_); + data_[size_++] = t; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const T& operator[] (size_t i) const { + eigen_assert(i < size_); + return data_[i]; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + T& operator[] (size_t i) { + eigen_assert(i < size_); + return data_[i]; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + T& back() { + eigen_assert(size_ > 0); + return data_[size_ - 1]; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const T& back() const { + eigen_assert(size_ > 0); + return data_[size_ - 1]; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + void pop_back() { + // NOTE: This does not destroy the value at the end the way + // std::vector's version of pop_back() does. That happens when + // the Vector is destroyed. + eigen_assert(size_ > 0); + size_--; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + size_t size() const { return size_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + bool empty() const { return size_ == 0; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + T* data() { return data_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const T* data() const { return data_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + T* begin() { return data_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + T* end() { return data_ + size_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const T* begin() const { return data_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const T* end() const { return data_ + size_; } + + private: + size_t reserve_; + size_t size_; + T* data_; +}; + +} // namespace Eigen + +#endif // EIGEN_FIXEDSIZEVECTOR_H diff --git a/unsupported/Eigen/CXX11/src/Tensor/README.md b/unsupported/Eigen/CXX11/src/Tensor/README.md index 407485090..eeca2f69e 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/README.md +++ b/unsupported/Eigen/CXX11/src/Tensor/README.md @@ -11,7 +11,7 @@ You can manipulate a tensor with one of the following classes. They all are in the namespace ```::Eigen.``` -### Class Tensor<data_type, rank> +### Class Tensor<data_type, rank> This is the class to use to create a tensor and allocate memory for it. The class is templatized with the tensor datatype, such as float or int, and the @@ -21,7 +21,7 @@ matrix. Tensors of this class are resizable. For example, if you assign a tensor of a different size to a Tensor, that tensor is resized to match its new value. -#### Constructor Tensor<data_type, rank>(size0, size1, ...) +#### Constructor Tensor<data_type, rank>(size0, size1, ...) Constructor for a Tensor. The constructor must be passed ```rank``` integers indicating the sizes of the instance along each of the the ```rank``` @@ -34,18 +34,18 @@ dimensions. // Resize t_3d by assigning a tensor of different sizes, but same rank. t_3d = Tensor<float, 3>(3, 4, 3); -#### Constructor Tensor<data_type, rank>(size_array) +#### Constructor Tensor<data_type, rank>(size_array) Constructor where the sizes for the constructor are specified as an array of values instead of an explicitly list of parameters. The array type to use is -```Eigen::array<Eigen::Index>```. The array can be constructed automatically +```Eigen::array<Eigen::Index>```. The array can be constructed automatically from an initializer list. // Create a tensor of strings of rank 2 with sizes 5, 7. Tensor<string, 2> t_2d({5, 7}); -### Class TensorFixedSize<data_type, Sizes<size0, size1, ...>> +### Class TensorFixedSize<data_type, Sizes<size0, size1, ...>> Class to use for tensors of fixed size, where the size is known at compile time. Fixed sized tensors can provide very fast computations because all their @@ -57,7 +57,7 @@ tensor data is held onto the stack and does not cause heap allocation and free. // Create a 4 x 3 tensor of floats. TensorFixedSize<float, Sizes<4, 3>> t_4x3; -### Class TensorMap<Tensor<data_type, rank>> +### Class TensorMap<Tensor<data_type, rank>> This is the class to use to create a tensor on top of memory allocated and owned by another part of your code. It allows to view any piece of allocated @@ -67,7 +67,7 @@ data are stored. A TensorMap is not resizable because it does not own the memory where its data are stored. -#### Constructor TensorMap<Tensor<data_type, rank>>(data, size0, size1, ...) +#### Constructor TensorMap<Tensor<data_type, rank>>(data, size0, size1, ...) Constructor for a Tensor. The constructor must be passed a pointer to the storage for the data, and "rank" size attributes. The storage has to be @@ -93,7 +93,7 @@ See Assigning to a TensorRef below. ## Accessing Tensor Elements -#### <data_type> tensor(index0, index1...) +#### <data_type> tensor(index0, index1...) Return the element at position ```(index0, index1...)``` in tensor ```tensor```. You must pass as many parameters as the rank of ```tensor```. @@ -175,7 +175,7 @@ the following code computes the elementwise addition of two tensors: While the code above looks easy enough, it is important to understand that the expression ```t1 + t2``` is not actually adding the values of the tensors. The expression instead constructs a "tensor operator" object of the class -TensorCwiseBinaryOp<scalar_sum>, which has references to the tensors +TensorCwiseBinaryOp<scalar_sum>, which has references to the tensors ```t1``` and ```t2```. This is a small C++ object that knows how to add ```t1``` and ```t2```. It is only when the value of the expression is assigned to the tensor ```t3``` that the addition is actually performed. Technically, @@ -452,24 +452,24 @@ memory for tensors with cuda. In the documentation of the tensor methods and Operation we mention datatypes that are tensor-type specific: -#### <Tensor-Type>::Dimensions +#### <Tensor-Type>::Dimensions Acts like an array of ints. Has an ```int size``` attribute, and can be indexed like an array to access individual values. Used to represent the dimensions of a tensor. See ```dimensions()```. -#### <Tensor-Type>::Index +#### <Tensor-Type>::Index Acts like an ```int```. Used for indexing tensors along their dimensions. See ```operator()```, ```dimension()```, and ```size()```. -#### <Tensor-Type>::Scalar +#### <Tensor-Type>::Scalar Represents the datatype of individual tensor elements. For example, for a ```Tensor<float>```, ```Scalar``` is the type ```float```. See ```setConstant()```. -#### <Operation> +#### <Operation> We use this pseudo type to indicate that a tensor Operation is returned by a method. We indicate in the text the type and dimensions of the tensor that the @@ -602,7 +602,7 @@ You can use one of the methods below to initialize the tensor memory. These have an immediate effect on the tensor and return the tensor itself as a result. These are not tensor Operations which delay evaluation. -### <Tensor-Type> setConstant(const Scalar& val) +### <Tensor-Type> setConstant(const Scalar& val) Sets all elements of the tensor to the constant value ```val```. ```Scalar``` is the type of data stored in the tensor. You can pass any value that is @@ -630,7 +630,7 @@ has a copy constructor and an ```operator=()```: yolo yolo yolo -### <Tensor-Type> setZero() +### <Tensor-Type> setZero() Fills the tensor with zeros. Equivalent to ```setConstant(Scalar(0))```. Returns the tensor itself in case you want to chain another call. @@ -644,7 +644,7 @@ Returns the tensor itself in case you want to chain another call. 0 0 0 0 -### <Tensor-Type> setValues({..initializer_list}) +### <Tensor-Type> setValues({..initializer_list}) Fills the tensor with explicit values specified in a std::initializer_list. The type of the initializer list depends on the type and rank of the tensor. @@ -680,7 +680,7 @@ code only sets the values of the first row of the tensor. 10 20 30 1000 1000 1000 -### <Tensor-Type> setRandom() +### <Tensor-Type> setRandom() Fills the tensor with random values. Returns the tensor itself in case you want to chain another call. @@ -775,7 +775,7 @@ The chain of Operation is evaluated lazily, typically when it is assigned to a tensor. See "Controlling when Expression are Evaluated" for more details about their evaluation. -### <Operation> constant(const Scalar& val) +### <Operation> constant(const Scalar& val) Returns a tensor of the same type and dimensions as the original tensor but where all elements have the value ```val```. @@ -803,7 +803,7 @@ tensor, or multiply every element of a tensor by a scalar. 0.6 0.6 0.6 0.6 0.6 0.6 -### <Operation> random() +### <Operation> random() Returns a tensor of the same type and dimensions as the current tensor but where all elements have random values. @@ -833,7 +833,7 @@ All these operations take a single input tensor as argument and return a tensor of the same type and dimensions as the tensor to which they are applied. The requested operations are applied to each element independently. -### <Operation> operator-() +### <Operation> operator-() Returns a tensor of the same type and dimensions as the original tensor containing the opposite values of the original tensor. @@ -852,42 +852,42 @@ containing the opposite values of the original tensor. -1 -1 -1 -1 -1 -1 -### <Operation> sqrt() +### <Operation> sqrt() Returns a tensor of the same type and dimensions as the original tensor containing the square roots of the original tensor. -### <Operation> rsqrt() +### <Operation> rsqrt() Returns a tensor of the same type and dimensions as the original tensor containing the inverse square roots of the original tensor. -### <Operation> square() +### <Operation> square() Returns a tensor of the same type and dimensions as the original tensor containing the squares of the original tensor values. -### <Operation> inverse() +### <Operation> inverse() Returns a tensor of the same type and dimensions as the original tensor containing the inverse of the original tensor values. -### <Operation> exp() +### <Operation> exp() Returns a tensor of the same type and dimensions as the original tensor containing the exponential of the original tensor. -### <Operation> log() +### <Operation> log() Returns a tensor of the same type and dimensions as the original tensor containing the natural logarithms of the original tensor. -### <Operation> abs() +### <Operation> abs() Returns a tensor of the same type and dimensions as the original tensor containing the absolute values of the original tensor. -### <Operation> pow(Scalar exponent) +### <Operation> pow(Scalar exponent) Returns a tensor of the same type and dimensions as the original tensor containing the coefficients of the original tensor to the power of the @@ -914,17 +914,17 @@ cubic roots of an int Tensor: 0 1 2 3 4 5 -### <Operation> operator * (Scalar scale) +### <Operation> operator * (Scalar scale) Multiplies all the coefficients of the input tensor by the provided scale. -### <Operation> cwiseMax(Scalar threshold) +### <Operation> cwiseMax(Scalar threshold) TODO -### <Operation> cwiseMin(Scalar threshold) +### <Operation> cwiseMin(Scalar threshold) TODO -### <Operation> unaryExpr(const CustomUnaryOp& func) +### <Operation> unaryExpr(const CustomUnaryOp& func) TODO @@ -936,39 +936,39 @@ dimensions as the tensors to which they are applied, and unless otherwise specified it is also of the same type. The requested operations are applied to each pair of elements independently. -### <Operation> operator+(const OtherDerived& other) +### <Operation> operator+(const OtherDerived& other) Returns a tensor of the same type and dimensions as the input tensors containing the coefficient wise sums of the inputs. -### <Operation> operator-(const OtherDerived& other) +### <Operation> operator-(const OtherDerived& other) Returns a tensor of the same type and dimensions as the input tensors containing the coefficient wise differences of the inputs. -### <Operation> operator*(const OtherDerived& other) +### <Operation> operator*(const OtherDerived& other) Returns a tensor of the same type and dimensions as the input tensors containing the coefficient wise products of the inputs. -### <Operation> operator/(const OtherDerived& other) +### <Operation> operator/(const OtherDerived& other) Returns a tensor of the same type and dimensions as the input tensors containing the coefficient wise quotients of the inputs. This operator is not supported for integer types. -### <Operation> cwiseMax(const OtherDerived& other) +### <Operation> cwiseMax(const OtherDerived& other) Returns a tensor of the same type and dimensions as the input tensors containing the coefficient wise maximums of the inputs. -### <Operation> cwiseMin(const OtherDerived& other) +### <Operation> cwiseMin(const OtherDerived& other) Returns a tensor of the same type and dimensions as the input tensors containing the coefficient wise mimimums of the inputs. -### <Operation> Logical operators +### <Operation> Logical operators The following logical operators are supported as well: @@ -1119,50 +1119,50 @@ one-dimension tensor with a single value. 276 -### <Operation> sum(const Dimensions& new_dims) -### <Operation> sum() +### <Operation> sum(const Dimensions& new_dims) +### <Operation> sum() Reduce a tensor using the sum() operator. The resulting values are the sum of the reduced values. -### <Operation> mean(const Dimensions& new_dims) -### <Operation> mean() +### <Operation> mean(const Dimensions& new_dims) +### <Operation> mean() Reduce a tensor using the mean() operator. The resulting values are the mean of the reduced values. -### <Operation> maximum(const Dimensions& new_dims) -### <Operation> maximum() +### <Operation> maximum(const Dimensions& new_dims) +### <Operation> maximum() Reduce a tensor using the maximum() operator. The resulting values are the largest of the reduced values. -### <Operation> minimum(const Dimensions& new_dims) -### <Operation> minimum() +### <Operation> minimum(const Dimensions& new_dims) +### <Operation> minimum() Reduce a tensor using the minimum() operator. The resulting values are the smallest of the reduced values. -### <Operation> prod(const Dimensions& new_dims) -### <Operation> prod() +### <Operation> prod(const Dimensions& new_dims) +### <Operation> prod() Reduce a tensor using the prod() operator. The resulting values are the product of the reduced values. -### <Operation> all(const Dimensions& new_dims) -### <Operation> all() +### <Operation> all(const Dimensions& new_dims) +### <Operation> all() Reduce a tensor using the all() operator. Casts tensor to bool and then checks whether all elements are true. Runs through all elements rather than short-circuiting, so may be significantly inefficient. -### <Operation> any(const Dimensions& new_dims) -### <Operation> any() +### <Operation> any(const Dimensions& new_dims) +### <Operation> any() Reduce a tensor using the any() operator. Casts tensor to bool and then checks whether any element is true. Runs through all elements rather than short-circuiting, so may be significantly inefficient. -### <Operation> reduce(const Dimensions& new_dims, const Reducer& reducer) +### <Operation> reduce(const Dimensions& new_dims, const Reducer& reducer) Reduce a tensor using a user-defined reduction operator. See ```SumReducer``` in TensorFunctors.h for information on how to implement a reduction operator. @@ -1170,7 +1170,7 @@ in TensorFunctors.h for information on how to implement a reduction operator. ## Convolutions -### <Operation> convolve(const Kernel& kernel, const Dimensions& dims) +### <Operation> convolve(const Kernel& kernel, const Dimensions& dims) Returns a tensor that is the output of the convolution of the input tensor with the kernel, along the specified dimensions of the input tensor. The dimension size for dimensions of the output tensor @@ -1213,7 +1213,7 @@ These operations return a Tensor with different dimensions than the original Tensor. They can be used to access slices of tensors, see them with different dimensions, or pad tensors with additional data. -### <Operation> reshape(const Dimensions& new_dims) +### <Operation> reshape(const Dimensions& new_dims) Returns a view of the input tensor that has been reshaped to the specified new dimensions. The argument new_dims is an array of Index values. The @@ -1292,7 +1292,7 @@ Note that "b" itself was not reshaped but that instead the assignment is done to the reshape view of b. -### <Operation> shuffle(const Shuffle& shuffle) +### <Operation> shuffle(const Shuffle& shuffle) Returns a copy of the input tensor whose dimensions have been reordered according to the specified permutation. The argument shuffle @@ -1333,7 +1333,7 @@ Let's rewrite the previous example to take advantage of this feature: output.shuffle({2, 0, 1}) = input; -### <Operation> stride(const Strides& strides) +### <Operation> stride(const Strides& strides) Returns a view of the input tensor that strides (skips stride-1 elements) along each of the dimensions. The argument strides is an @@ -1359,7 +1359,7 @@ It is possible to assign a tensor to a stride: output.stride({2, 3, 4}) = input; -### <Operation> slice(const StartIndices& offsets, const Sizes& extents) +### <Operation> slice(const StartIndices& offsets, const Sizes& extents) Returns a sub-tensor of the given tensor. For each dimension i, the slice is made of the coefficients stored between offset[i] and offset[i] + extents[i] in @@ -1385,7 +1385,7 @@ the input tensor. 600 700 -### <Operation> chip(const Index offset, const Index dim) +### <Operation> chip(const Index offset, const Index dim) A chip is a special kind of slice. It is the subtensor at the given offset in the dimension dim. The returned tensor has one fewer dimension than the input @@ -1436,7 +1436,7 @@ lvalue. For example: 0 0 0 -### <Operation> reverse(const ReverseDimensions& reverse) +### <Operation> reverse(const ReverseDimensions& reverse) Returns a view of the input tensor that reverses the order of the coefficients along a subset of the dimensions. The argument reverse is an array of boolean @@ -1466,7 +1466,7 @@ of a 2D tensor: 0 100 200 -### <Operation> broadcast(const Broadcast& broadcast) +### <Operation> broadcast(const Broadcast& broadcast) Returns a view of the input tensor in which the input is replicated one to many times. @@ -1490,11 +1490,11 @@ made in each of the dimensions. 0 100 200 0 100 200 300 400 500 300 400 500 -### <Operation> concatenate(const OtherDerived& other, Axis axis) +### <Operation> concatenate(const OtherDerived& other, Axis axis) TODO -### <Operation> pad(const PaddingDimensions& padding) +### <Operation> pad(const PaddingDimensions& padding) Returns a view of the input tensor in which the input is padded with zeros. @@ -1519,7 +1519,7 @@ Returns a view of the input tensor in which the input is padded with zeros. 0 0 0 0 -### <Operation> extract_patches(const PatchDims& patch_dims) +### <Operation> extract_patches(const PatchDims& patch_dims) Returns a tensor of coefficient patches extracted from the input tensor, where each patch is of dimension specified by 'patch_dims'. The returned tensor has @@ -1606,7 +1606,7 @@ patch index: 5 6 7 10 11 -### <Operation> extract_image_patches(const Index patch_rows, const Index patch_cols, +### <Operation> extract_image_patches(const Index patch_rows, const Index patch_cols, const Index row_stride, const Index col_stride, const PaddingType padding_type) @@ -1663,7 +1663,7 @@ sizes: ## Special Operations -### <Operation> cast<T>() +### <Operation> cast<T>() Returns a tensor of type T with the same dimensions as the original tensor. The returned tensor contains the values of the original tensor converted to @@ -1692,7 +1692,7 @@ but you can easily cast the tensors to floats to do the division: 1 2 2 -### <Operation> eval() +### <Operation> eval() TODO @@ -1701,7 +1701,7 @@ TODO Scalar values are often represented by tensors of size 1 and rank 1. It would be more logical and user friendly to use tensors of rank 0 instead. For example -Tensor<T, N>::maximum() currently returns a Tensor<T, 1>. Similarly, the inner +Tensor<T, N>::maximum() currently returns a Tensor<T, 1>. Similarly, the inner product of 2 1d tensors (through contractions) returns a 1d tensor. In the future these operations might be updated to return 0d tensors instead. diff --git a/unsupported/Eigen/CXX11/src/Tensor/Tensor.h b/unsupported/Eigen/CXX11/src/Tensor/Tensor.h index 6d357545c..759dede3f 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/Tensor.h +++ b/unsupported/Eigen/CXX11/src/Tensor/Tensor.h @@ -69,16 +69,14 @@ class Tensor : public TensorBase<Tensor<Scalar_, NumIndices_, Options_, IndexTyp typedef typename internal::traits<Self>::StorageKind StorageKind; typedef typename internal::traits<Self>::Index Index; typedef Scalar_ Scalar; - typedef typename internal::packet_traits<Scalar>::type Packet; typedef typename NumTraits<Scalar>::Real RealScalar; typedef typename Base::CoeffReturnType CoeffReturnType; - typedef typename Base::PacketReturnType PacketReturnType; enum { IsAligned = bool(EIGEN_MAX_ALIGN_BYTES>0) & !(Options_&DontAlign), - PacketAccess = (internal::packet_traits<Scalar>::size > 1), Layout = Options_ & RowMajor ? RowMajor : ColMajor, CoordAccess = true, + RawAccess = true }; static const int Options = Options_; @@ -340,42 +338,42 @@ class Tensor : public TensorBase<Tensor<Scalar_, NumIndices_, Options_, IndexTyp #ifdef EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> - inline Tensor(Index firstDimension, IndexTypes... otherDimensions) - : m_storage(internal::array_prod(array<Index, NumIndices>{{firstDimension, otherDimensions...}}), array<Index, NumIndices>{{firstDimension, otherDimensions...}}) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Tensor(Index firstDimension, IndexTypes... otherDimensions) + : m_storage(firstDimension, otherDimensions...) { // The number of dimensions used to construct a tensor must be equal to the rank of the tensor. EIGEN_STATIC_ASSERT(sizeof...(otherDimensions) + 1 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) } #else - inline explicit Tensor(Index dim1) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Tensor(Index dim1) : m_storage(dim1, array<Index, 1>(dim1)) { EIGEN_STATIC_ASSERT(1 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) } - inline explicit Tensor(Index dim1, Index dim2) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Tensor(Index dim1, Index dim2) : m_storage(dim1*dim2, array<Index, 2>(dim1, dim2)) { EIGEN_STATIC_ASSERT(2 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) } - inline explicit Tensor(Index dim1, Index dim2, Index dim3) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Tensor(Index dim1, Index dim2, Index dim3) : m_storage(dim1*dim2*dim3, array<Index, 3>(dim1, dim2, dim3)) { EIGEN_STATIC_ASSERT(3 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) } - inline explicit Tensor(Index dim1, Index dim2, Index dim3, Index dim4) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Tensor(Index dim1, Index dim2, Index dim3, Index dim4) : m_storage(dim1*dim2*dim3*dim4, array<Index, 4>(dim1, dim2, dim3, dim4)) { EIGEN_STATIC_ASSERT(4 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) } - inline explicit Tensor(Index dim1, Index dim2, Index dim3, Index dim4, Index dim5) - : m_storage(dim1*dim2*dim3*dim4*dim5, array<Index, 4>(dim1, dim2, dim3, dim4, dim5)) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Tensor(Index dim1, Index dim2, Index dim3, Index dim4, Index dim5) + : m_storage(dim1*dim2*dim3*dim4*dim5, array<Index, 5>(dim1, dim2, dim3, dim4, dim5)) { EIGEN_STATIC_ASSERT(5 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) } #endif /** Normal Dimension */ - inline explicit Tensor(const array<Index, NumIndices>& dimensions) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit Tensor(const array<Index, NumIndices>& dimensions) : m_storage(internal::array_prod(dimensions), dimensions) { EIGEN_INITIALIZE_COEFFS_IF_THAT_OPTION_IS_ENABLED diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorArgMax.h b/unsupported/Eigen/CXX11/src/Tensor/TensorArgMax.h index c783aab97..f1ec04c49 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorArgMax.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorArgMax.h @@ -89,6 +89,7 @@ struct TensorEvaluator<const TensorIndexTupleOp<ArgType>, Device> BlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -134,7 +135,7 @@ struct traits<TensorTupleReducerOp<ReduceOp, Dims, XprType> > : public traits<Xp typedef Index Scalar; typedef typename XprType::Nested Nested; typedef typename remove_reference<Nested>::type _Nested; - static const int NumDimensions = XprTraits::NumDimensions; + static const int NumDimensions = XprTraits::NumDimensions - array_size<Dims>::value; static const int Layout = XprTraits::Layout; }; @@ -210,6 +211,7 @@ struct TensorEvaluator<const TensorTupleReducerOp<ReduceOp, Dims, ArgType>, Devi BlockAccess = false, Layout = TensorEvaluator<const TensorReductionOp<ReduceOp, Dims, const TensorIndexTupleOp<ArgType> >, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorAssign.h b/unsupported/Eigen/CXX11/src/Tensor/TensorAssign.h index a41d4d265..199d2ce41 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorAssign.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorAssign.h @@ -25,7 +25,6 @@ template<typename LhsXprType, typename RhsXprType> struct traits<TensorAssignOp<LhsXprType, RhsXprType> > { typedef typename LhsXprType::Scalar Scalar; - typedef typename internal::packet_traits<Scalar>::type Packet; typedef typename traits<LhsXprType>::StorageKind StorageKind; typedef typename promote_index_type<typename traits<LhsXprType>::Index, typename traits<RhsXprType>::Index>::type Index; @@ -62,10 +61,8 @@ class TensorAssignOp : public TensorBase<TensorAssignOp<LhsXprType, RhsXprType> { public: typedef typename Eigen::internal::traits<TensorAssignOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorAssignOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename LhsXprType::CoeffReturnType CoeffReturnType; - typedef typename LhsXprType::PacketReturnType PacketReturnType; typedef typename Eigen::internal::nested<TensorAssignOp>::type Nested; typedef typename Eigen::internal::traits<TensorAssignOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorAssignOp>::Index Index; @@ -97,6 +94,7 @@ struct TensorEvaluator<const TensorAssignOp<LeftArgType, RightArgType>, Device> IsAligned = TensorEvaluator<LeftArgType, Device>::IsAligned & TensorEvaluator<RightArgType, Device>::IsAligned, PacketAccess = TensorEvaluator<LeftArgType, Device>::PacketAccess & TensorEvaluator<RightArgType, Device>::PacketAccess, Layout = TensorEvaluator<LeftArgType, Device>::Layout, + RawAccess = TensorEvaluator<LeftArgType, Device>::RawAccess, }; EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) : @@ -109,7 +107,7 @@ struct TensorEvaluator<const TensorAssignOp<LeftArgType, RightArgType>, Device> typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef typename TensorEvaluator<RightArgType, Device>::Dimensions Dimensions; EIGEN_DEVICE_FUNC const Dimensions& dimensions() const @@ -152,6 +150,8 @@ struct TensorEvaluator<const TensorAssignOp<LeftArgType, RightArgType>, Device> return m_leftImpl.template packet<LoadMode>(index); } + EIGEN_DEVICE_FUNC CoeffReturnType* data() const { return m_leftImpl.data(); } + private: TensorEvaluator<LeftArgType, Device> m_leftImpl; TensorEvaluator<RightArgType, Device> m_rightImpl; diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorBase.h b/unsupported/Eigen/CXX11/src/Tensor/TensorBase.h index d1ce3d0ed..69d1802d5 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorBase.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorBase.h @@ -31,7 +31,6 @@ class TensorBase<Derived, ReadOnlyAccessors> typedef typename DerivedTraits::Scalar Scalar; typedef typename DerivedTraits::Index Index; typedef typename internal::remove_const<Scalar>::type CoeffReturnType; - typedef typename internal::packet_traits<CoeffReturnType>::type PacketReturnType; static const int NumDimensions = DerivedTraits::NumDimensions; // Generic nullary operation support. @@ -123,6 +122,58 @@ class TensorBase<Derived, ReadOnlyAccessors> } EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_lgamma_op<Scalar>, const Derived> + lgamma() const { + return unaryExpr(internal::scalar_lgamma_op<Scalar>()); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_digamma_op<Scalar>, const Derived> + digamma() const { + return unaryExpr(internal::scalar_digamma_op<Scalar>()); + } + + // igamma(a = this, x = other) + template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const TensorCwiseBinaryOp<internal::scalar_igamma_op<Scalar>, const Derived, const OtherDerived> + igamma(const OtherDerived& other) const { + return binaryExpr(other.derived(), internal::scalar_igamma_op<Scalar>()); + } + + // igammac(a = this, x = other) + template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const TensorCwiseBinaryOp<internal::scalar_igammac_op<Scalar>, const Derived, const OtherDerived> + igammac(const OtherDerived& other) const { + return binaryExpr(other.derived(), internal::scalar_igammac_op<Scalar>()); + } + + // zeta(x = this, q = other) + template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const TensorCwiseBinaryOp<internal::scalar_zeta_op<Scalar>, const Derived, const OtherDerived> + zeta(const OtherDerived& other) const { + return binaryExpr(other.derived(), internal::scalar_zeta_op<Scalar>()); + } + + // polygamma(n = this, x = other) + template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const TensorCwiseBinaryOp<internal::scalar_polygamma_op<Scalar>, const Derived, const OtherDerived> + polygamma(const OtherDerived& other) const { + return binaryExpr(other.derived(), internal::scalar_polygamma_op<Scalar>()); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_erf_op<Scalar>, const Derived> + erf() const { + return unaryExpr(internal::scalar_erf_op<Scalar>()); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_erfc_op<Scalar>, const Derived> + erfc() const { + return unaryExpr(internal::scalar_erfc_op<Scalar>()); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_sigmoid_op<Scalar>, const Derived> sigmoid() const { return unaryExpr(internal::scalar_sigmoid_op<Scalar>()); @@ -147,6 +198,12 @@ class TensorBase<Derived, ReadOnlyAccessors> } EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_conjugate_op<Scalar>, const Derived> + conjugate() const { + return unaryExpr(internal::scalar_conjugate_op<Scalar>()); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_pow_op<Scalar>, const Derived> pow(Scalar exponent) const { return unaryExpr(internal::scalar_pow_op<Scalar>(exponent)); @@ -202,6 +259,25 @@ class TensorBase<Derived, ReadOnlyAccessors> return TensorConversionOp<NewType, const Derived>(derived()); } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_round_op<Scalar>, const Derived> + round() const { + return unaryExpr(internal::scalar_round_op<Scalar>()); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_ceil_op<Scalar>, const Derived> + ceil() const { + return unaryExpr(internal::scalar_ceil_op<Scalar>()); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_floor_op<Scalar>, const Derived> + floor() const { + return unaryExpr(internal::scalar_floor_op<Scalar>()); + } + + // Generic binary operation support. template <typename CustomBinaryOp, typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<CustomBinaryOp, const Derived, const OtherDerived> @@ -285,6 +361,7 @@ class TensorBase<Derived, ReadOnlyAccessors> operator==(const OtherDerived& other) const { return binaryExpr(other.derived(), internal::scalar_cmp_op<Scalar, internal::cmp_EQ>()); } + template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorCwiseBinaryOp<internal::scalar_cmp_op<Scalar, internal::cmp_NEQ>, const Derived, const OtherDerived> operator!=(const OtherDerived& other) const { @@ -323,6 +400,23 @@ class TensorBase<Derived, ReadOnlyAccessors> return operator!=(constant(threshold)); } + // Checks + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_isnan_op<Scalar>, const Derived> + (isnan)() const { + return unaryExpr(internal::scalar_isnan_op<Scalar>()); + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_isinf_op<Scalar>, const Derived> + (isinf)() const { + return unaryExpr(internal::scalar_isinf_op<Scalar>()); + } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const TensorCwiseUnaryOp<internal::scalar_isfinite_op<Scalar>, const Derived> + (isfinite)() const { + return unaryExpr(internal::scalar_isfinite_op<Scalar>()); + } + // Coefficient-wise ternary operators. template<typename ThenDerived, typename ElseDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorSelectOp<const Derived, const ThenDerived, const ElseDerived> @@ -544,7 +638,7 @@ class TensorBase<Derived, ReadOnlyAccessors> const TensorVolumePatchOp<Dynamic, Dynamic, Dynamic, const Derived> extract_volume_patches(const Index patch_planes, const Index patch_rows, const Index patch_cols, const Index plane_stride = 1, const Index row_stride = 1, const Index col_stride = 1, - const PaddingType padding_type = PADDING_SAME, const Scalar padding_value = 0) const { + const PaddingType padding_type = PADDING_SAME, const Scalar padding_value = Scalar(0)) const { return TensorVolumePatchOp<Dynamic, Dynamic, Dynamic, const Derived>(derived(), patch_planes, patch_rows, patch_cols, plane_stride, row_stride, col_stride, 1, 1, 1, 1, 1, 1, padding_type, padding_value); } @@ -556,7 +650,7 @@ class TensorBase<Derived, ReadOnlyAccessors> const Index plane_inflate_stride, const Index row_inflate_stride, const Index col_inflate_stride, const Index padding_top_z, const Index padding_bottom_z, const Index padding_top, const Index padding_bottom, - const Index padding_left, const Index padding_right, const Scalar padding_value = 0) const { + const Index padding_left, const Index padding_right, const Scalar padding_value = Scalar(0)) const { return TensorVolumePatchOp<Dynamic, Dynamic, Dynamic, const Derived>(derived(), patch_planes, patch_rows, patch_cols, plane_stride, row_stride, col_stride, 1, 1, 1, plane_inflate_stride, row_inflate_stride, col_inflate_stride, padding_top_z, padding_bottom_z, padding_top, padding_bottom, padding_left, padding_right, padding_value); } @@ -594,7 +688,12 @@ class TensorBase<Derived, ReadOnlyAccessors> template <typename PaddingDimensions> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorPaddingOp<const PaddingDimensions, const Derived> pad(const PaddingDimensions& padding) const { - return TensorPaddingOp<const PaddingDimensions, const Derived>(derived(), padding); + return TensorPaddingOp<const PaddingDimensions, const Derived>(derived(), padding, internal::scalar_cast_op<int, Scalar>()(0)); + } + template <typename PaddingDimensions> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + const TensorPaddingOp<const PaddingDimensions, const Derived> + pad(const PaddingDimensions& padding, const Scalar padding_value) const { + return TensorPaddingOp<const PaddingDimensions, const Derived>(derived(), padding, padding_value); } template <typename Shuffle> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const TensorShufflingOp<const Shuffle, const Derived> @@ -652,7 +751,6 @@ class TensorBase<Derived, WriteAccessors> : public TensorBase<Derived, ReadOnlyA typedef typename DerivedTraits::Scalar Scalar; typedef typename DerivedTraits::Index Index; typedef Scalar CoeffReturnType; - typedef typename internal::packet_traits<Scalar>::type PacketReturnType; static const int NumDimensions = DerivedTraits::NumDimensions; template <typename Scalar, int NumIndices, int Options, typename IndexType> friend class Tensor; diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorBroadcasting.h b/unsupported/Eigen/CXX11/src/Tensor/TensorBroadcasting.h index dc64959e1..b6e6db12a 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorBroadcasting.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorBroadcasting.h @@ -25,7 +25,6 @@ struct traits<TensorBroadcastingOp<Broadcast, XprType> > : public traits<XprType { typedef typename XprType::Scalar Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprTraits::StorageKind StorageKind; typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; @@ -46,6 +45,21 @@ struct nested<TensorBroadcastingOp<Broadcast, XprType>, 1, typename eval<TensorB typedef TensorBroadcastingOp<Broadcast, XprType> type; }; +template <typename Dims> +struct is_input_scalar { + static const bool value = false; +}; +template <> +struct is_input_scalar<Sizes<> > { + static const bool value = true; +}; +#ifndef EIGEN_EMULATE_CXX11_META_H +template <typename std::size_t... Indices> +struct is_input_scalar<Sizes<Indices...> > { + static const bool value = (Sizes<Indices...>::total_size == 1); +}; +#endif + } // end namespace internal @@ -55,10 +69,8 @@ class TensorBroadcastingOp : public TensorBase<TensorBroadcastingOp<Broadcast, X { public: typedef typename Eigen::internal::traits<TensorBroadcastingOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorBroadcastingOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; typedef typename Eigen::internal::nested<TensorBroadcastingOp>::type Nested; typedef typename Eigen::internal::traits<TensorBroadcastingOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorBroadcastingOp>::Index Index; @@ -94,6 +106,7 @@ struct TensorEvaluator<const TensorBroadcastingOp<Broadcast, ArgType>, Device> IsAligned = false, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -103,7 +116,7 @@ struct TensorEvaluator<const TensorBroadcastingOp<Broadcast, ArgType>, Device> // and store the result in a scalar. Instead one should reshape the scalar into a a N-D // tensor with N >= 1 of 1 element first and then broadcast. EIGEN_STATIC_ASSERT(NumDims > 0, YOU_MADE_A_PROGRAMMING_MISTAKE); - const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions(); + const InputDimensions& input_dims = m_impl.dimensions(); const Broadcast& broadcast = op.broadcast(); for (int i = 0; i < NumDims; ++i) { eigen_assert(input_dims[i] > 0); @@ -128,7 +141,7 @@ struct TensorEvaluator<const TensorBroadcastingOp<Broadcast, ArgType>, Device> } typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -143,6 +156,10 @@ struct TensorEvaluator<const TensorBroadcastingOp<Broadcast, ArgType>, Device> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE CoeffReturnType coeff(Index index) const { + if (internal::is_input_scalar<typename internal::remove_all<InputDimensions>::type>::value) { + return m_impl.coeff(0); + } + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { return coeffColMajor(index); } else { @@ -214,6 +231,10 @@ struct TensorEvaluator<const TensorBroadcastingOp<Broadcast, ArgType>, Device> template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketReturnType packet(Index index) const { + if (internal::is_input_scalar<typename internal::remove_all<InputDimensions>::type>::value) { + return internal::pset1<PacketReturnType>(m_impl.coeff(0)); + } + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { return packetColMajor<LoadMode>(index); } else { diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorChipping.h b/unsupported/Eigen/CXX11/src/Tensor/TensorChipping.h index c9fa39e51..c21a98fe0 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorChipping.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorChipping.h @@ -26,7 +26,6 @@ struct traits<TensorChippingOp<DimId, XprType> > : public traits<XprType> { typedef typename XprType::Scalar Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprTraits::StorageKind StorageKind; typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; @@ -50,7 +49,7 @@ struct nested<TensorChippingOp<DimId, XprType>, 1, typename eval<TensorChippingO template <DenseIndex DimId> struct DimensionId { - DimensionId(DenseIndex dim) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DimensionId(DenseIndex dim) { eigen_assert(dim == DimId); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DenseIndex actualDim() const { @@ -60,7 +59,7 @@ struct DimensionId template <> struct DimensionId<Dynamic> { - DimensionId(DenseIndex dim) : actual_dim(dim) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DimensionId(DenseIndex dim) : actual_dim(dim) { eigen_assert(dim >= 0); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DenseIndex actualDim() const { @@ -80,10 +79,8 @@ class TensorChippingOp : public TensorBase<TensorChippingOp<DimId, XprType> > { public: typedef typename Eigen::internal::traits<TensorChippingOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorChippingOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; typedef typename Eigen::internal::nested<TensorChippingOp>::type Nested; typedef typename Eigen::internal::traits<TensorChippingOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorChippingOp>::Index Index; @@ -145,6 +142,7 @@ struct TensorEvaluator<const TensorChippingOp<DimId, ArgType>, Device> PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -183,7 +181,7 @@ struct TensorEvaluator<const TensorChippingOp<DimId, ArgType>, Device> } typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -244,8 +242,8 @@ struct TensorEvaluator<const TensorChippingOp<DimId, ArgType>, Device> } } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar* data() const { - Scalar* result = m_impl.data(); + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType* data() const { + CoeffReturnType* result = const_cast<CoeffReturnType*>(m_impl.data()); if (((static_cast<int>(Layout) == static_cast<int>(ColMajor) && m_dim.actualDim() == NumDims) || (static_cast<int>(Layout) == static_cast<int>(RowMajor) && m_dim.actualDim() == 0)) && result) { @@ -304,6 +302,7 @@ struct TensorEvaluator<TensorChippingOp<DimId, ArgType>, Device> enum { IsAligned = false, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -311,7 +310,7 @@ struct TensorEvaluator<TensorChippingOp<DimId, ArgType>, Device> { } typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(Index index) { diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorConcatenation.h b/unsupported/Eigen/CXX11/src/Tensor/TensorConcatenation.h index 3d153bb94..7738f18fb 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorConcatenation.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorConcatenation.h @@ -26,7 +26,6 @@ struct traits<TensorConcatenationOp<Axis, LhsXprType, RhsXprType> > // Type promotion to handle the case where the types of the lhs and the rhs are different. typedef typename promote_storage_type<typename LhsXprType::Scalar, typename RhsXprType::Scalar>::ret Scalar; - typedef typename packet_traits<Scalar>::type Packet; typedef typename promote_storage_type<typename traits<LhsXprType>::StorageKind, typename traits<RhsXprType>::StorageKind>::ret StorageKind; typedef typename promote_index_type<typename traits<LhsXprType>::Index, @@ -60,14 +59,11 @@ class TensorConcatenationOp : public TensorBase<TensorConcatenationOp<Axis, LhsX { public: typedef typename internal::traits<TensorConcatenationOp>::Scalar Scalar; - typedef typename internal::traits<TensorConcatenationOp>::Packet Packet; typedef typename internal::traits<TensorConcatenationOp>::StorageKind StorageKind; typedef typename internal::traits<TensorConcatenationOp>::Index Index; typedef typename internal::nested<TensorConcatenationOp>::type Nested; typedef typename internal::promote_storage_type<typename LhsXprType::CoeffReturnType, typename RhsXprType::CoeffReturnType>::ret CoeffReturnType; - typedef typename internal::promote_storage_type<typename LhsXprType::PacketReturnType, - typename RhsXprType::PacketReturnType>::ret PacketReturnType; typedef typename NumTraits<Scalar>::Real RealScalar; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorConcatenationOp(const LhsXprType& lhs, const RhsXprType& rhs, Axis axis) @@ -120,11 +116,12 @@ struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgTy typedef DSizes<Index, NumDims> Dimensions; typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; enum { IsAligned = false, PacketAccess = TensorEvaluator<LeftArgType, Device>::PacketAccess & TensorEvaluator<RightArgType, Device>::PacketAccess, Layout = TensorEvaluator<LeftArgType, Device>::Layout, + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -287,6 +284,7 @@ template<typename Axis, typename LeftArgType, typename RightArgType, typename De IsAligned = false, PacketAccess = TensorEvaluator<LeftArgType, Device>::PacketAccess & TensorEvaluator<RightArgType, Device>::PacketAccess, Layout = TensorEvaluator<LeftArgType, Device>::Layout, + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(XprType& op, const Device& device) @@ -298,7 +296,7 @@ template<typename Axis, typename LeftArgType, typename RightArgType, typename De typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(Index index) { diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContraction.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContraction.h index eda93a1de..f070ba61e 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorContraction.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorContraction.h @@ -21,365 +21,12 @@ namespace Eigen { */ namespace internal { -enum { - Rhs = 0, - Lhs = 1, -}; - -/* - * Implementation of the Eigen blas_data_mapper class for tensors. - */ -template<typename Scalar, typename Index, int side, - typename Tensor, - typename nocontract_t, typename contract_t, - int packet_size, bool inner_dim_contiguous> -class SimpleTensorContractionMapper { - public: - EIGEN_DEVICE_FUNC - SimpleTensorContractionMapper(const Tensor& tensor, - const nocontract_t& nocontract_strides, - const nocontract_t& ij_strides, - const contract_t& contract_strides, - const contract_t& k_strides) : - m_tensor(tensor), - m_nocontract_strides(nocontract_strides), - m_ij_strides(ij_strides), - m_contract_strides(contract_strides), - m_k_strides(k_strides) { } - - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE void prefetch(Index /*i*/) { } - - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Scalar operator()(Index row) const { - // column major assumption - return operator()(row, 0); - } - - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Scalar operator()(Index row, Index col) const { - return m_tensor.coeff(computeIndex(row, col)); - } - - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Index computeIndex(Index row, Index col) const { - const bool left = (side == Lhs); - Index nocontract_val = left ? row : col; - Index linidx = 0; - for (int i = static_cast<int>(array_size<nocontract_t>::value) - 1; i > 0; i--) { - const Index idx = nocontract_val / m_ij_strides[i]; - linidx += idx * m_nocontract_strides[i]; - nocontract_val -= idx * m_ij_strides[i]; - } - if (array_size<typename Tensor::Dimensions>::value > array_size<contract_t>::value) { - if (side == Lhs && inner_dim_contiguous) { - eigen_assert(m_nocontract_strides[0] == 1); - linidx += nocontract_val; - } else { - linidx += nocontract_val * m_nocontract_strides[0]; - } - } - - Index contract_val = left ? col : row; - for (int i = static_cast<int>(array_size<contract_t>::value) - 1; i > 0; i--) { - const Index idx = contract_val / m_k_strides[i]; - linidx += idx * m_contract_strides[i]; - contract_val -= idx * m_k_strides[i]; - } - - if(array_size<contract_t>::value > 0) { - if (side == Rhs && inner_dim_contiguous) { - eigen_assert(m_contract_strides[0] == 1); - linidx += contract_val; - } else { - linidx += contract_val * m_contract_strides[0]; - } - } - - return linidx; - } - - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE IndexPair<Index> computeIndexPair(Index row, Index col, const Index distance) const { - const bool left = (side == Lhs); - Index nocontract_val[2] = {left ? row : col, left ? row + distance : col}; - Index linidx[2] = {0, 0}; - for (int i = static_cast<int>(array_size<nocontract_t>::value) - 1; i > 0; i--) { - const Index idx0 = nocontract_val[0] / m_ij_strides[i]; - const Index idx1 = nocontract_val[1] / m_ij_strides[i]; - linidx[0] += idx0 * m_nocontract_strides[i]; - linidx[1] += idx1 * m_nocontract_strides[i]; - nocontract_val[0] -= idx0 * m_ij_strides[i]; - nocontract_val[1] -= idx1 * m_ij_strides[i]; - } - if (array_size<typename Tensor::Dimensions>::value > array_size<contract_t>::value) { - if (side == Lhs && inner_dim_contiguous) { - eigen_assert(m_nocontract_strides[0] == 1); - linidx[0] += nocontract_val[0]; - linidx[1] += nocontract_val[1]; - } else { - linidx[0] += nocontract_val[0] * m_nocontract_strides[0]; - linidx[1] += nocontract_val[1] * m_nocontract_strides[0]; - } - } - - Index contract_val[2] = {left ? col : row, left ? col : row + distance}; - for (int i = static_cast<int>(array_size<contract_t>::value) - 1; i > 0; i--) { - const Index idx0 = contract_val[0] / m_k_strides[i]; - const Index idx1 = contract_val[1] / m_k_strides[i]; - linidx[0] += idx0 * m_contract_strides[i]; - linidx[1] += idx1 * m_contract_strides[i]; - contract_val[0] -= idx0 * m_k_strides[i]; - contract_val[1] -= idx1 * m_k_strides[i]; - } - - if (side == Rhs && inner_dim_contiguous) { - eigen_assert(m_contract_strides[0] == 1); - linidx[0] += contract_val[0]; - linidx[1] += contract_val[1]; - } else { - linidx[0] += contract_val[0] * m_contract_strides[0]; - linidx[1] += contract_val[1] * m_contract_strides[0]; - } - return IndexPair<Index>(linidx[0], linidx[1]); - } - - Index firstAligned(Index size) const { - return size; - } - Index stride() const { - return 1; - } - - protected: - const Tensor m_tensor; - const nocontract_t m_nocontract_strides; - const nocontract_t m_ij_strides; - const contract_t m_contract_strides; - const contract_t m_k_strides; -}; - - -template<typename Scalar, typename Index, int side, - typename Tensor, - typename nocontract_t, typename contract_t, - int packet_size, bool inner_dim_contiguous, - bool inner_dim_reordered, int Alignment> - class BaseTensorContractionMapper : public SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous> -{ - public: - typedef SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous> ParentMapper; - - EIGEN_DEVICE_FUNC - BaseTensorContractionMapper(const Tensor& tensor, - const nocontract_t& nocontract_strides, - const nocontract_t& ij_strides, - const contract_t& contract_strides, - const contract_t& k_strides) : - ParentMapper(tensor, nocontract_strides, ij_strides, contract_strides, k_strides) { } - - typedef typename packet_traits<Scalar>::type Packet; - typedef typename packet_traits<Scalar>::half HalfPacket; - - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Packet loadPacket(Index i, Index j) const { - // whole method makes column major assumption - - // don't need to add offsets for now (because operator handles that) - // current code assumes packet size must be a multiple of 2 - EIGEN_STATIC_ASSERT(packet_size % 2 == 0, YOU_MADE_A_PROGRAMMING_MISTAKE); - - if (Tensor::PacketAccess && inner_dim_contiguous && !inner_dim_reordered) { - const Index index = this->computeIndex(i, j); - eigen_assert(this->computeIndex(i+packet_size-1, j) == index + packet_size-1); - return this->m_tensor.template packet<Alignment>(index); - } - - const IndexPair<Index> indexPair = this->computeIndexPair(i, j, packet_size - 1); - const Index first = indexPair.first; - const Index last = indexPair.second; - - // We can always do optimized packet reads from left hand side right now, because - // the vertical matrix dimension on the left hand side is never contracting. - // On the right hand side we need to check if the contracting dimensions may have - // been shuffled first. - if (Tensor::PacketAccess && - (side == Lhs || internal::array_size<contract_t>::value <= 1 || !inner_dim_reordered) && - (last - first) == (packet_size - 1)) { - - return this->m_tensor.template packet<Alignment>(first); - } - - EIGEN_ALIGN_MAX Scalar data[packet_size]; - - data[0] = this->m_tensor.coeff(first); - for (Index k = 1; k < packet_size - 1; k += 2) { - const IndexPair<Index> internal_pair = this->computeIndexPair(i + k, j, 1); - data[k] = this->m_tensor.coeff(internal_pair.first); - data[k + 1] = this->m_tensor.coeff(internal_pair.second); - } - data[packet_size - 1] = this->m_tensor.coeff(last); - - return pload<Packet>(data); - } - - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE HalfPacket loadHalfPacket(Index i, Index j) const { - // whole method makes column major assumption - - // don't need to add offsets for now (because operator handles that) - const Index half_packet_size = unpacket_traits<HalfPacket>::size; - if (half_packet_size == packet_size) { - return loadPacket(i, j); - } - EIGEN_ALIGN_MAX Scalar data[half_packet_size]; - for (Index k = 0; k < half_packet_size; k++) { - data[k] = operator()(i + k, j); - } - return pload<HalfPacket>(data); - } -}; - - -template<typename Scalar, typename Index, int side, - typename Tensor, - typename nocontract_t, typename contract_t, - bool inner_dim_contiguous, - bool inner_dim_reordered, int Alignment> -class BaseTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, 1, inner_dim_contiguous, inner_dim_reordered, Alignment> : public SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, 1, inner_dim_contiguous> -{ - public: - typedef SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, 1, inner_dim_contiguous> ParentMapper; - - EIGEN_DEVICE_FUNC - BaseTensorContractionMapper(const Tensor& tensor, - const nocontract_t& nocontract_strides, - const nocontract_t& ij_strides, - const contract_t& contract_strides, - const contract_t& k_strides) : - ParentMapper(tensor, nocontract_strides, ij_strides, contract_strides, k_strides) { } - - typedef typename packet_traits<Scalar>::type Packet; - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Packet loadPacket(Index i, Index j) const { - EIGEN_ALIGN_MAX Scalar data[1]; - data[0] = this->m_tensor.coeff(this->computeIndex(i, j)); - return pload<typename packet_traits<Scalar>::type>(data); - } - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Packet loadHalfPacket(Index i, Index j) const { - return loadPacket(i, j); - } -}; - -template<typename Scalar, typename Index, int side, - typename Tensor, - typename nocontract_t, typename contract_t, - int packet_size, - bool inner_dim_contiguous, bool inner_dim_reordered, int Alignment> -class TensorContractionInputMapper; - -template<typename Scalar, typename Index, int side, - typename Tensor, - typename nocontract_t, typename contract_t, - int packet_size, - bool inner_dim_contiguous, bool inner_dim_reordered, int Alignment> -class TensorContractionSubMapper { - public: - typedef typename packet_traits<Scalar>::type Packet; - typedef typename packet_traits<Scalar>::half HalfPacket; - - typedef TensorContractionInputMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment> ParentMapper; - typedef TensorContractionSubMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment> Self; - typedef Self LinearMapper; - - EIGEN_DEVICE_FUNC TensorContractionSubMapper(const ParentMapper& base_mapper, Index vert_offset, Index horiz_offset) - : m_base_mapper(base_mapper), m_vert_offset(vert_offset), m_horiz_offset(horiz_offset) { } - - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar operator()(Index i) const { - return m_base_mapper(i + m_vert_offset, m_horiz_offset); - } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar operator()(Index i, Index j) const { - return m_base_mapper(i + m_vert_offset, j + m_horiz_offset); - } - - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet loadPacket(Index i) const { - return m_base_mapper.loadPacket(i + m_vert_offset, m_horiz_offset); - } - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet loadPacket(Index i, Index j) const { - return m_base_mapper.loadPacket(i + m_vert_offset, j + m_horiz_offset); - } - - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE HalfPacket loadHalfPacket(Index i) const { - return m_base_mapper.loadHalfPacket(i + m_vert_offset, m_horiz_offset); - } - - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, Packet p) const { - m_base_mapper.storePacket(i + m_vert_offset, m_horiz_offset, p); - } - - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE LinearMapper getLinearMapper(Index i, Index j) const { - return LinearMapper(m_base_mapper, i + m_vert_offset, j + m_horiz_offset); - } - - template <typename PacketT, int AlignmentType> - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT load(Index i) const { - EIGEN_STATIC_ASSERT((internal::is_same<PacketT, Packet>::value), YOU_MADE_A_PROGRAMMING_MISTAKE); - EIGEN_STATIC_ASSERT((AlignmentType == Aligned || Alignment == Unaligned), YOU_MADE_A_PROGRAMMING_MISTAKE); - return loadPacket(i); - } - - template <typename Packet> - EIGEN_DEVICE_FUNC bool aligned(Index) const { - return false; - } - - private: - const ParentMapper& m_base_mapper; - const Index m_vert_offset; - const Index m_horiz_offset; -}; - - -template<typename Scalar, typename Index, int side, - typename Tensor, - typename nocontract_t, typename contract_t, - int packet_size, - bool inner_dim_contiguous, bool inner_dim_reordered, int Alignment> -class TensorContractionInputMapper - : public BaseTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment> { - - public: - typedef BaseTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment> Base; - typedef TensorContractionSubMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment> SubMapper; - typedef SubMapper VectorMapper; - - EIGEN_DEVICE_FUNC TensorContractionInputMapper(const Tensor& tensor, - const nocontract_t& nocontract_strides, - const nocontract_t& ij_strides, - const contract_t& contract_strides, - const contract_t& k_strides) - : Base(tensor, nocontract_strides, ij_strides, contract_strides, k_strides) { } - - EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE SubMapper getSubMapper(Index i, Index j) const { - return SubMapper(*this, i, j); - } - - EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE VectorMapper getVectorMapper(Index i, Index j) const { - return VectorMapper(*this, i, j); - } -}; - - - template<typename Dimensions, typename LhsXprType, typename RhsXprType> struct traits<TensorContractionOp<Dimensions, LhsXprType, RhsXprType> > { // Type promotion to handle the case where the types of the lhs and the rhs are different. typedef typename internal::promote_storage_type<typename LhsXprType::Scalar, typename RhsXprType::Scalar>::ret Scalar; - typedef typename internal::packet_traits<Scalar>::type Packet; typedef typename promote_storage_type<typename traits<LhsXprType>::StorageKind, typename traits<RhsXprType>::StorageKind>::ret StorageKind; typedef typename promote_index_type<typename traits<LhsXprType>::Index, @@ -428,11 +75,8 @@ class TensorContractionOp : public TensorBase<TensorContractionOp<Indices, LhsXp { public: typedef typename Eigen::internal::traits<TensorContractionOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorContractionOp>::Packet Packet; typedef typename internal::promote_storage_type<typename LhsXprType::CoeffReturnType, typename RhsXprType::CoeffReturnType>::ret CoeffReturnType; - typedef typename internal::promote_storage_type<typename LhsXprType::PacketReturnType, - typename RhsXprType::PacketReturnType>::ret PacketReturnType; typedef typename Eigen::internal::nested<TensorContractionOp>::type Nested; typedef typename Eigen::internal::traits<TensorContractionOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorContractionOp>::Index Index; @@ -470,16 +114,16 @@ struct TensorContractionEvaluatorBase typedef TensorContractionOp<Indices, LeftArgType, RightArgType> XprType; typedef typename internal::remove_const<typename XprType::Scalar>::type Scalar; - typedef typename XprType::Packet Packet; typedef typename XprType::Index Index; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; enum { IsAligned = true, - PacketAccess = (internal::packet_traits<Scalar>::size > 1), + PacketAccess = (internal::unpacket_traits<PacketReturnType>::size > 1), Layout = TensorEvaluator<LeftArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = true }; // Most of the code is assuming that both input tensors are ColMajor. If the @@ -498,8 +142,6 @@ struct TensorContractionEvaluatorBase static const int ContractDims = internal::array_size<Indices>::value; static const int NumDims = max_n_1<LDims + RDims - 2 * ContractDims>::size; - typedef array<Index, LDims> left_dim_mapper_t; - typedef array<Index, RDims> right_dim_mapper_t; typedef array<Index, ContractDims> contract_t; typedef array<Index, max_n_1<LDims - ContractDims>::size> left_nocontract_t; typedef array<Index, max_n_1<RDims - ContractDims>::size> right_nocontract_t; @@ -546,8 +188,21 @@ struct TensorContractionEvaluatorBase // We need to flip all the pairs of contracting indices as well as // reversing the dimensions. for (int i = 0; i < ContractDims; i++) { - eval_op_indices[i].first = LDims - 1 - op.indices()[i].second; - eval_op_indices[i].second = RDims - 1 - op.indices()[i].first; + eval_op_indices[i].first = LDims - 1 - op.indices()[ContractDims - 1 - i].second; + eval_op_indices[i].second = RDims - 1 - op.indices()[ContractDims - 1 - i].first; + } + } + + // Check for duplicate axes and make sure the first index in eval_op_indices + // is increasing. Using O(n^2) sorting is OK since ContractDims is small + for (int i = 0; i < ContractDims; i++) { + for (int j = i + 1; j < ContractDims; j++) { + eigen_assert(eval_op_indices[j].first != eval_op_indices[i].first && + eval_op_indices[j].second != eval_op_indices[i].second && + "contraction axes should be unique"); + if (eval_op_indices[j].first < eval_op_indices[i].first) { + numext::swap(eval_op_indices[j], eval_op_indices[i]); + } } } @@ -731,7 +386,7 @@ struct TensorContractionEvaluatorBase } template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment> - void evalGemv(Scalar* buffer) const { + EIGEN_DEVICE_FUNC void evalGemv(Scalar* buffer) const { const Index rows = m_i_size; const Index cols = m_k_size; @@ -739,19 +394,21 @@ struct TensorContractionEvaluatorBase typedef typename internal::remove_const<typename EvalRightArgType::Scalar>::type RhsScalar; typedef TensorEvaluator<EvalLeftArgType, Device> LeftEvaluator; typedef TensorEvaluator<EvalRightArgType, Device> RightEvaluator; - const Index lhs_packet_size = internal::packet_traits<LhsScalar>::size; - const Index rhs_packet_size = internal::packet_traits<RhsScalar>::size; + const Index lhs_packet_size = internal::unpacket_traits<typename LeftEvaluator::PacketReturnType>::size; + const Index rhs_packet_size = internal::unpacket_traits<typename RightEvaluator::PacketReturnType>::size; + const int lhs_alignment = LeftEvaluator::IsAligned ? Aligned : Unaligned; + const int rhs_alignment = RightEvaluator::IsAligned ? Aligned : Unaligned; typedef internal::TensorContractionInputMapper<LhsScalar, Index, internal::Lhs, LeftEvaluator, left_nocontract_t, contract_t, lhs_packet_size, lhs_inner_dim_contiguous, - false, Unaligned> LhsMapper; + false, lhs_alignment> LhsMapper; typedef internal::TensorContractionInputMapper<RhsScalar, Index, internal::Rhs, RightEvaluator, right_nocontract_t, contract_t, rhs_packet_size, rhs_inner_dim_contiguous, - rhs_inner_dim_reordered, Unaligned> RhsMapper; + rhs_inner_dim_reordered, rhs_alignment> RhsMapper; LhsMapper lhs(m_leftImpl, m_left_nocontract_strides, m_i_strides, m_left_contracting_strides, m_k_strides); @@ -784,11 +441,11 @@ struct TensorContractionEvaluatorBase } template<int LoadMode> - EIGEN_DEVICE_FUNC PacketReturnType packet(Index index) const { - return internal::ploadt<Packet, LoadMode>(m_result + index); + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { + return internal::ploadt<PacketReturnType, LoadMode>(m_result + index); } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar* data() const { return m_result; } protected: // Prevent assignment @@ -829,10 +486,9 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT typedef TensorContractionOp<Indices, LeftArgType, RightArgType> XprType; typedef typename internal::remove_const<typename XprType::Scalar>::type Scalar; - typedef typename XprType::Packet Packet; typedef typename XprType::Index Index; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; enum { Layout = TensorEvaluator<LeftArgType, Device>::Layout, @@ -853,9 +509,6 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT internal::array_size<typename TensorEvaluator<EvalRightArgType, Device>::Dimensions>::value; static const int ContractDims = internal::array_size<Indices>::value; - typedef array<Index, LDims> left_dim_mapper_t; - typedef array<Index, RDims> right_dim_mapper_t; - typedef array<Index, ContractDims> contract_t; typedef array<Index, max_n_1<LDims - ContractDims>::size> left_nocontract_t; typedef array<Index, max_n_1<RDims - ContractDims>::size> right_nocontract_t; @@ -870,7 +523,7 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT Base(op, device) { } template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment> - void evalProduct(Scalar* buffer) const { + EIGEN_DEVICE_FUNC void evalProduct(Scalar* buffer) const { if (this->m_j_size == 1) { this->template evalGemv<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered, Alignment>(buffer); return; @@ -904,8 +557,8 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT typedef TensorEvaluator<EvalLeftArgType, Device> LeftEvaluator; typedef TensorEvaluator<EvalRightArgType, Device> RightEvaluator; - const Index lhs_packet_size = internal::packet_traits<LhsScalar>::size; - const Index rhs_packet_size = internal::packet_traits<RhsScalar>::size; + const Index lhs_packet_size = internal::unpacket_traits<typename LeftEvaluator::PacketReturnType>::size; + const Index rhs_packet_size = internal::unpacket_traits<typename RightEvaluator::PacketReturnType>::size; typedef internal::TensorContractionInputMapper<LhsScalar, Index, internal::Lhs, LeftEvaluator, left_nocontract_t, @@ -936,10 +589,8 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT OutputMapper output(buffer, m); - typedef typename internal::gemm_blocking_space<ColMajor, LhsScalar, RhsScalar, Dynamic, Dynamic, Dynamic> BlockingType; - // Sizes of the blocks to load in cache. See the Goto paper for details. - BlockingType blocking(m, n, k, 1, true); + internal::TensorContractionBlocking<LhsMapper, RhsMapper, Index, internal::ShardByCol> blocking(k, m, n, 1); const Index kc = blocking.kc(); const Index mc = numext::mini(m, blocking.mc()); const Index nc = numext::mini(n, blocking.nc()); diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionBlocking.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionBlocking.h new file mode 100644 index 000000000..3d3f6904f --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionBlocking.h @@ -0,0 +1,58 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_BLOCKING_H +#define EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_BLOCKING_H + + +namespace Eigen { +namespace internal { + +enum { + ShardByRow = 0, + ShardByCol = 1 +}; + + +// Default Blocking Strategy +template <typename LhsMapper, typename RhsMapper, typename Index, int ShardingType=ShardByCol> +class TensorContractionBlocking { + public: + + typedef typename LhsMapper::Scalar LhsScalar; + typedef typename RhsMapper::Scalar RhsScalar; + + EIGEN_DEVICE_FUNC TensorContractionBlocking(Index k, Index m, Index n, Index num_threads = 1) : + kc_(k), mc_(m), nc_(n) + { + if (ShardingType == ShardByCol) { + computeProductBlockingSizes<LhsScalar, RhsScalar, 1>(kc_, mc_, nc_, num_threads); + } + else { + if (kc_ && mc_ && nc_) { + mc_ = (((m / num_threads) + 15) / 16) * 16; + } + } + } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index kc() const { return kc_; } + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index mc() const { return mc_; } + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index nc() const { return nc_; } + + private: + Index kc_; + Index mc_; + Index nc_; +}; + + +} // end namespace internal +} // end namespace Eigen + +#endif // EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_BLOCKING_H diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionCuda.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionCuda.h index 90ee50678..dbff660a9 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionCuda.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionCuda.h @@ -20,7 +20,7 @@ template<typename Scalar, typename Index, typename LhsMapper, typename RhsMapper, typename OutputMapper, bool needs_edge_check> __device__ EIGEN_STRONG_INLINE void EigenContractionKernelInternal(const LhsMapper lhs, const RhsMapper rhs, - const OutputMapper output, volatile Scalar* lhs_shmem, volatile Scalar* rhs_shmem, + const OutputMapper output, Scalar* lhs_shmem, Scalar* rhs_shmem, const Index m_size, const Index n_size, const Index k_size) { const Index m_block_idx = blockIdx.x; @@ -99,23 +99,23 @@ EigenContractionKernelInternal(const LhsMapper lhs, const RhsMapper rhs, #define prefetchIntoRegisters(base_k) \ { \ - lhs_pf0 = Scalar(0); \ - lhs_pf1 = Scalar(0); \ - lhs_pf2 = Scalar(0); \ - lhs_pf3 = Scalar(0); \ - lhs_pf4 = Scalar(0); \ - lhs_pf5 = Scalar(0); \ - lhs_pf6 = Scalar(0); \ - lhs_pf7 = Scalar(0); \ + lhs_pf0 = conv(0); \ + lhs_pf1 = conv(0); \ + lhs_pf2 = conv(0); \ + lhs_pf3 = conv(0); \ + lhs_pf4 = conv(0); \ + lhs_pf5 = conv(0); \ + lhs_pf6 = conv(0); \ + lhs_pf7 = conv(0); \ \ - rhs_pf0 = Scalar(0); \ - rhs_pf1 = Scalar(0); \ - rhs_pf2 = Scalar(0); \ - rhs_pf3 = Scalar(0); \ - rhs_pf4 = Scalar(0); \ - rhs_pf5 = Scalar(0); \ - rhs_pf6 = Scalar(0); \ - rhs_pf7 = Scalar(0); \ + rhs_pf0 = conv(0); \ + rhs_pf1 = conv(0); \ + rhs_pf2 = conv(0); \ + rhs_pf3 = conv(0); \ + rhs_pf4 = conv(0); \ + rhs_pf5 = conv(0); \ + rhs_pf6 = conv(0); \ + rhs_pf7 = conv(0); \ \ if (!needs_edge_check || lhs_vert < m_size) { \ const Index lhs_horiz_0 = base_k + threadIdx.z + 0 * 8; \ @@ -261,15 +261,16 @@ EigenContractionKernelInternal(const LhsMapper lhs, const RhsMapper rhs, // declare and initialize result array #define res(i, j) _res_##i##j #define initResultRow(i) \ - Scalar res(i, 0) = Scalar(0); \ - Scalar res(i, 1) = Scalar(0); \ - Scalar res(i, 2) = Scalar(0); \ - Scalar res(i, 3) = Scalar(0); \ - Scalar res(i, 4) = Scalar(0); \ - Scalar res(i, 5) = Scalar(0); \ - Scalar res(i, 6) = Scalar(0); \ - Scalar res(i, 7) = Scalar(0); \ - + Scalar res(i, 0) = conv(0); \ + Scalar res(i, 1) = conv(0); \ + Scalar res(i, 2) = conv(0); \ + Scalar res(i, 3) = conv(0); \ + Scalar res(i, 4) = conv(0); \ + Scalar res(i, 5) = conv(0); \ + Scalar res(i, 6) = conv(0); \ + Scalar res(i, 7) = conv(0); \ + + internal::scalar_cast_op<int, Scalar> conv; initResultRow(0); initResultRow(1); initResultRow(2); @@ -318,8 +319,8 @@ EigenContractionKernelInternal(const LhsMapper lhs, const RhsMapper rhs, Scalar rrow(7); // Now x corresponds to k, y to m, and z to n - const volatile Scalar* lhs_block = &lhs_shmem[threadIdx.x + 9 * threadIdx.y]; - const volatile Scalar* rhs_block = &rhs_shmem[threadIdx.x + 8 * threadIdx.z]; + const Scalar* lhs_block = &lhs_shmem[threadIdx.x + 9 * threadIdx.y]; + const Scalar* rhs_block = &rhs_shmem[threadIdx.x + 8 * threadIdx.z]; #define lhs_element(i, j) lhs_block[72 * ((i) + 8 * (j))] #define rhs_element(i, j) rhs_block[72 * ((i) + 8 * (j))] @@ -502,8 +503,8 @@ __launch_bounds__(512) EigenContractionKernel(const LhsMapper lhs, const RhsMapper rhs, const OutputMapper output, const Index m_size, const Index n_size, const Index k_size) { - __shared__ volatile Scalar lhs_shmem[72 * 64]; - __shared__ volatile Scalar rhs_shmem[72 * 64]; + __shared__ Scalar lhs_shmem[72 * 64]; + __shared__ Scalar rhs_shmem[72 * 64]; const Index m_block_idx = blockIdx.x; const Index n_block_idx = blockIdx.y; @@ -1212,10 +1213,9 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT typedef TensorContractionOp<Indices, LeftArgType, RightArgType> XprType; typedef typename internal::remove_const<typename XprType::Scalar>::type Scalar; - typedef typename XprType::Packet Packet; typedef typename XprType::Index Index; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, GpuDevice>::type PacketReturnType; enum { Layout = TensorEvaluator<LeftArgType, Device>::Layout, @@ -1261,7 +1261,7 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT Base(op, device) {} // We need to redefine this method to make nvcc happy - EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* data) { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* data) { this->m_leftImpl.evalSubExprsIfNeeded(NULL); this->m_rightImpl.evalSubExprsIfNeeded(NULL); if (data) { @@ -1313,10 +1313,39 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT } } + template <typename LhsScalar, typename RhsScalar, typename Index, typename LhsMapper, typename RhsMapper, typename OutputMapper> struct LaunchKernels { + static void Run(const LhsMapper& lhs, const RhsMapper& rhs, const OutputMapper& output, Index m, Index n, Index k, const GpuDevice& device) { + const Index m_blocks = (m + 63) / 64; + const Index n_blocks = (n + 63) / 64; + const dim3 num_blocks(m_blocks, n_blocks, 1); + const dim3 block_size(8, 8, 8); + LAUNCH_CUDA_KERNEL((EigenContractionKernel<Scalar, Index, LhsMapper, RhsMapper, OutputMapper>), num_blocks, block_size, 0, device, lhs, rhs, output, m, n, k); + } + }; + + template <typename Index, typename LhsMapper, typename RhsMapper, typename OutputMapper> struct LaunchKernels<float, float, Index, LhsMapper, RhsMapper, OutputMapper> { + static void Run(const LhsMapper& lhs, const RhsMapper& rhs, const OutputMapper& output, Index m, Index n, Index k, const GpuDevice& device) { + if (m < 768 || n < 768) { + const Index m_blocks = (m + 63) / 64; + const Index n_blocks = (n + 63) / 64; + const dim3 num_blocks(m_blocks, n_blocks, 1); + const dim3 block_size(16, 16, 1); + LAUNCH_CUDA_KERNEL((EigenFloatContractionKernel16x16<Index, LhsMapper, RhsMapper, OutputMapper>), num_blocks, block_size, 0, device, lhs, rhs, output, m, n, k); + } else { + const Index m_blocks = (m + 127) / 128; + const Index n_blocks = (n + 63) / 64; + const dim3 num_blocks(m_blocks, n_blocks, 1); + const dim3 block_size(8, 32, 1); + LAUNCH_CUDA_KERNEL((EigenFloatContractionKernel<Index, LhsMapper, RhsMapper, OutputMapper>), num_blocks, block_size, 0, device, lhs, rhs, output, m, n, k); + } + } + }; + template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment> void evalTyped(Scalar* buffer) const { // columns in left side, rows in right side const Index k = this->m_k_size; + EIGEN_UNUSED_VARIABLE(k) // rows in left side const Index m = this->m_i_size; @@ -1352,28 +1381,7 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT OutputMapper output(buffer, m); setCudaSharedMemConfig(cudaSharedMemBankSizeEightByte); - if (internal::is_same<LhsScalar, float>::value && - internal::is_same<RhsScalar, float>::value) { - if (m < 768 || n < 768) { - const Index m_blocks = (m + 63) / 64; - const Index n_blocks = (n + 63) / 64; - const dim3 num_blocks(m_blocks, n_blocks, 1); - const dim3 block_size(16, 16, 1); - LAUNCH_CUDA_KERNEL((EigenFloatContractionKernel16x16<Index, LhsMapper, RhsMapper, OutputMapper>), num_blocks, block_size, 0, this->m_device, lhs, rhs, output, m, n, k); - } else { - const Index m_blocks = (m + 127) / 128; - const Index n_blocks = (n + 63) / 64; - const dim3 num_blocks(m_blocks, n_blocks, 1); - const dim3 block_size(8, 32, 1); - LAUNCH_CUDA_KERNEL((EigenFloatContractionKernel<Index, LhsMapper, RhsMapper, OutputMapper>), num_blocks, block_size, 0, this->m_device, lhs, rhs, output, m, n, k); - } - } else { - const Index m_blocks = (m + 63) / 64; - const Index n_blocks = (n + 63) / 64; - const dim3 num_blocks(m_blocks, n_blocks, 1); - const dim3 block_size(8, 8, 8); - LAUNCH_CUDA_KERNEL((EigenContractionKernel<Scalar, Index, LhsMapper, RhsMapper, OutputMapper>), num_blocks, block_size, 0, this->m_device, lhs, rhs, output, m, n, k); - } + LaunchKernels<LhsScalar, RhsScalar, Index, LhsMapper, RhsMapper, OutputMapper>::Run(lhs, rhs, output, m, n, k, this->m_device); } }; diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionMapper.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionMapper.h new file mode 100644 index 000000000..392cb6e3d --- /dev/null +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionMapper.h @@ -0,0 +1,465 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_MAPPER_H +#define EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_MAPPER_H + +namespace Eigen { + +namespace internal { + +enum { + Rhs = 0, + Lhs = 1, +}; + +/* + * Implementation of the Eigen blas_data_mapper class for tensors. + */ + +template <typename Tensor, bool HasRawAccess> struct CoeffLoader { + enum { + DirectOffsets = false + }; + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE CoeffLoader(const Tensor& tensor) : m_tensor(tensor) { } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void offsetBuffer(typename Tensor::Index) { + eigen_assert(false && "unsupported"); + } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE typename Tensor::Scalar coeff(typename Tensor::Index index) const { return m_tensor.coeff(index); } + + template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + typename Tensor::PacketReturnType packet(typename Tensor::Index index) const + { + return m_tensor.template packet<LoadMode>(index); + } + + + private: + const Tensor m_tensor; +}; + +template <typename Tensor> struct CoeffLoader<Tensor, true> { + enum { + DirectOffsets = true + }; + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE CoeffLoader(const Tensor& tensor) : m_data(tensor.data()) {} + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void offsetBuffer(typename Tensor::Index offset) { + m_data += offset; + } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE typename Tensor::Scalar coeff(typename Tensor::Index index) const { return loadConstant(m_data+index); } + + template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + typename Tensor::PacketReturnType packet(typename Tensor::Index index) const + { + return internal::ploadt_ro<typename Tensor::PacketReturnType, LoadMode>(m_data + index); + } + private: + typedef typename Tensor::Scalar Scalar; + const Scalar* m_data; +}; + +template<typename Scalar, typename Index, int side, + typename Tensor, + typename nocontract_t, typename contract_t, + int packet_size, bool inner_dim_contiguous, int Alignment> +class SimpleTensorContractionMapper { + public: + EIGEN_DEVICE_FUNC + SimpleTensorContractionMapper(const Tensor& tensor, + const nocontract_t& nocontract_strides, + const nocontract_t& ij_strides, + const contract_t& contract_strides, + const contract_t& k_strides) : + m_tensor(tensor), + m_nocontract_strides(nocontract_strides), + m_ij_strides(ij_strides), + m_contract_strides(contract_strides), + m_k_strides(k_strides) { } + + enum { + DirectOffsets = CoeffLoader<Tensor, Tensor::RawAccess>::DirectOffsets + }; + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void offsetBuffer(typename Tensor::Index offset) { + m_tensor.offsetBuffer(offset); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE void prefetch(Index /*i*/) { } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Scalar operator()(Index row) const { + // column major assumption + return operator()(row, 0); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Scalar operator()(Index row, Index col) const { + return m_tensor.coeff(computeIndex(row, col)); + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Index computeIndex(Index row, Index col) const { + const bool left = (side == Lhs); + Index nocontract_val = left ? row : col; + Index linidx = 0; + for (int i = static_cast<int>(array_size<nocontract_t>::value) - 1; i > 0; i--) { + const Index idx = nocontract_val / m_ij_strides[i]; + linidx += idx * m_nocontract_strides[i]; + nocontract_val -= idx * m_ij_strides[i]; + } + if (array_size<typename Tensor::Dimensions>::value > array_size<contract_t>::value) { + if (side == Lhs && inner_dim_contiguous) { + eigen_assert(m_nocontract_strides[0] == 1); + linidx += nocontract_val; + } else { + linidx += nocontract_val * m_nocontract_strides[0]; + } + } + + Index contract_val = left ? col : row; + for (int i = static_cast<int>(array_size<contract_t>::value) - 1; i > 0; i--) { + const Index idx = contract_val / m_k_strides[i]; + linidx += idx * m_contract_strides[i]; + contract_val -= idx * m_k_strides[i]; + } + + if(array_size<contract_t>::value > 0) { + if (side == Rhs && inner_dim_contiguous) { + eigen_assert(m_contract_strides[0] == 1); + linidx += contract_val; + } else { + linidx += contract_val * m_contract_strides[0]; + } + } + + return linidx; + } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE IndexPair<Index> computeIndexPair(Index row, Index col, const Index distance) const { + const bool left = (side == Lhs); + Index nocontract_val[2] = {left ? row : col, left ? row + distance : col}; + Index linidx[2] = {0, 0}; + for (int i = static_cast<int>(array_size<nocontract_t>::value) - 1; i > 0; i--) { + const Index idx0 = nocontract_val[0] / m_ij_strides[i]; + const Index idx1 = nocontract_val[1] / m_ij_strides[i]; + linidx[0] += idx0 * m_nocontract_strides[i]; + linidx[1] += idx1 * m_nocontract_strides[i]; + nocontract_val[0] -= idx0 * m_ij_strides[i]; + nocontract_val[1] -= idx1 * m_ij_strides[i]; + } + if (array_size<typename Tensor::Dimensions>::value > array_size<contract_t>::value) { + if (side == Lhs && inner_dim_contiguous) { + eigen_assert(m_nocontract_strides[0] == 1); + linidx[0] += nocontract_val[0]; + linidx[1] += nocontract_val[1]; + } else { + linidx[0] += nocontract_val[0] * m_nocontract_strides[0]; + linidx[1] += nocontract_val[1] * m_nocontract_strides[0]; + } + } + + Index contract_val[2] = {left ? col : row, left ? col : row + distance}; + for (int i = static_cast<int>(array_size<contract_t>::value) - 1; i > 0; i--) { + const Index idx0 = contract_val[0] / m_k_strides[i]; + const Index idx1 = contract_val[1] / m_k_strides[i]; + linidx[0] += idx0 * m_contract_strides[i]; + linidx[1] += idx1 * m_contract_strides[i]; + contract_val[0] -= idx0 * m_k_strides[i]; + contract_val[1] -= idx1 * m_k_strides[i]; + } + + if (side == Rhs && inner_dim_contiguous) { + eigen_assert(m_contract_strides[0] == 1); + linidx[0] += contract_val[0]; + linidx[1] += contract_val[1]; + } else { + linidx[0] += contract_val[0] * m_contract_strides[0]; + linidx[1] += contract_val[1] * m_contract_strides[0]; + } + return IndexPair<Index>(linidx[0], linidx[1]); + } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index firstAligned(Index size) const { + // Only claim alignment when we can compute the actual stride (ie when we're + // dealing with the lhs with inner_dim_contiguous. This is because the + // matrix-vector product relies on the stride when dealing with aligned inputs. + return (Alignment == Aligned) && (side == Lhs) && inner_dim_contiguous ? 0 : size; + } + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index stride() const { + return ((side == Lhs) && inner_dim_contiguous) ? m_contract_strides[0] : 1; + } + + protected: + CoeffLoader<Tensor, Tensor::RawAccess> m_tensor; + const nocontract_t m_nocontract_strides; + const nocontract_t m_ij_strides; + const contract_t m_contract_strides; + const contract_t m_k_strides; +}; + + +template<typename Scalar, typename Index, int side, + typename Tensor, + typename nocontract_t, typename contract_t, + int packet_size, bool inner_dim_contiguous, + bool inner_dim_reordered, int Alignment> +class BaseTensorContractionMapper : public SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, Alignment> +{ + public: + typedef SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, Alignment> ParentMapper; + + EIGEN_DEVICE_FUNC + BaseTensorContractionMapper(const Tensor& tensor, + const nocontract_t& nocontract_strides, + const nocontract_t& ij_strides, + const contract_t& contract_strides, + const contract_t& k_strides) : + ParentMapper(tensor, nocontract_strides, ij_strides, contract_strides, k_strides) { } + + typedef typename Tensor::PacketReturnType Packet; + typedef typename unpacket_traits<Packet>::half HalfPacket; + + template <int AlignmentType = Alignment> + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Packet loadPacket(Index i, Index j) const { + // whole method makes column major assumption + + // don't need to add offsets for now (because operator handles that) + // current code assumes packet size must be a multiple of 2 + EIGEN_STATIC_ASSERT(packet_size % 2 == 0, YOU_MADE_A_PROGRAMMING_MISTAKE); + + if (Tensor::PacketAccess && inner_dim_contiguous && !inner_dim_reordered) { + const Index index = this->computeIndex(i, j); + eigen_assert(this->computeIndex(i+packet_size-1, j) == index + packet_size-1); + return this->m_tensor.template packet<AlignmentType>(index); + } + + const IndexPair<Index> indexPair = this->computeIndexPair(i, j, packet_size - 1); + const Index first = indexPair.first; + const Index last = indexPair.second; + + // We can always do optimized packet reads from left hand side right now, because + // the vertical matrix dimension on the left hand side is never contracting. + // On the right hand side we need to check if the contracting dimensions may have + // been shuffled first. + if (Tensor::PacketAccess && + (side == Lhs || internal::array_size<contract_t>::value <= 1 || !inner_dim_reordered) && + (last - first) == (packet_size - 1)) { + + return this->m_tensor.template packet<AlignmentType>(first); + } + + EIGEN_ALIGN_MAX Scalar data[packet_size]; + + data[0] = this->m_tensor.coeff(first); + for (Index k = 1; k < packet_size - 1; k += 2) { + const IndexPair<Index> internal_pair = this->computeIndexPair(i + k, j, 1); + data[k] = this->m_tensor.coeff(internal_pair.first); + data[k + 1] = this->m_tensor.coeff(internal_pair.second); + } + data[packet_size - 1] = this->m_tensor.coeff(last); + + return pload<Packet>(data); + } + + template <int AlignmentType = Alignment> + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE HalfPacket loadHalfPacket(Index i, Index j) const { + // whole method makes column major assumption + + // don't need to add offsets for now (because operator handles that) + const Index half_packet_size = unpacket_traits<HalfPacket>::size; + if (half_packet_size == packet_size) { + return loadPacket<AlignmentType>(i, j); + } + EIGEN_ALIGN_MAX Scalar data[half_packet_size]; + for (Index k = 0; k < half_packet_size; k++) { + data[k] = operator()(i + k, j); + } + return pload<HalfPacket>(data); + } +}; + + +template<typename Scalar, typename Index, int side, + typename Tensor, + typename nocontract_t, typename contract_t, + bool inner_dim_contiguous, + bool inner_dim_reordered, int Alignment> +class BaseTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, 1, inner_dim_contiguous, inner_dim_reordered, Alignment> : public SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, 1, inner_dim_contiguous, Alignment> +{ + public: + typedef SimpleTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, 1, inner_dim_contiguous, Alignment> ParentMapper; + + EIGEN_DEVICE_FUNC + BaseTensorContractionMapper(const Tensor& tensor, + const nocontract_t& nocontract_strides, + const nocontract_t& ij_strides, + const contract_t& contract_strides, + const contract_t& k_strides) : + ParentMapper(tensor, nocontract_strides, ij_strides, contract_strides, k_strides) { } + + typedef typename Tensor::PacketReturnType Packet; + template <int> EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Packet loadPacket(Index i, Index j) const { + EIGEN_ALIGN_MAX Scalar data[1]; + data[0] = this->m_tensor.coeff(this->computeIndex(i, j)); + return pload<typename Tensor::PacketReturnType>(data); + } + template <int> EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Packet loadHalfPacket(Index i, Index j) const { + return loadPacket(i, j); + } +}; + + +template<typename Scalar, typename Index, int side, + typename Tensor, + typename nocontract_t, typename contract_t, + int packet_size, + bool inner_dim_contiguous, bool inner_dim_reordered, int Alignment> +class TensorContractionSubMapper { + public: + typedef typename Tensor::PacketReturnType Packet; + typedef typename unpacket_traits<Packet>::half HalfPacket; + + typedef BaseTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment> ParentMapper; + typedef TensorContractionSubMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment> Self; + typedef Self LinearMapper; + + enum { + // We can use direct offsets iff the parent mapper supports then and we can compute the strides. + // TODO: we should also enable direct offsets for the Rhs case. + UseDirectOffsets = ParentMapper::DirectOffsets && (side == Lhs) && inner_dim_contiguous && (array_size<contract_t>::value > 0) + }; + + EIGEN_DEVICE_FUNC TensorContractionSubMapper(const ParentMapper& base_mapper, Index vert_offset, Index horiz_offset) + : m_base_mapper(base_mapper), m_vert_offset(vert_offset), m_horiz_offset(horiz_offset) { + // Bake the offsets into the buffer used by the base mapper whenever possible. This avoids the need to recompute + // this offset every time we attempt to access a coefficient. + if (UseDirectOffsets) { + Index stride = m_base_mapper.stride(); + m_base_mapper.offsetBuffer(vert_offset + horiz_offset * stride); + } + } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar operator()(Index i) const { + if (UseDirectOffsets) { + return m_base_mapper(i, 0); + } + return m_base_mapper(i + m_vert_offset, m_horiz_offset); + } + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar operator()(Index i, Index j) const { + if (UseDirectOffsets) { + return m_base_mapper(i, j); + } + return m_base_mapper(i + m_vert_offset, j + m_horiz_offset); + } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet loadPacket(Index i) const { + if (UseDirectOffsets) { + return m_base_mapper.template loadPacket<Alignment>(i, 0); + } + return m_base_mapper.template loadPacket<Alignment>(i + m_vert_offset, m_horiz_offset); + } + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet loadPacket(Index i, Index j) const { + if (UseDirectOffsets) { + return m_base_mapper.template loadPacket<Alignment>(i, j); + } + return m_base_mapper.template loadPacket<Alignment>(i + m_vert_offset, j + m_horiz_offset); + } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE HalfPacket loadHalfPacket(Index i) const { + if (UseDirectOffsets) { + return m_base_mapper.template loadHalfPacket<Alignment>(i, 0); + } + return m_base_mapper.template loadHalfPacket<Alignment>(i + m_vert_offset, m_horiz_offset); + } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void storePacket(Index i, Packet p) const { + if (UseDirectOffsets) { + m_base_mapper.storePacket(i, 0, p); + } + m_base_mapper.storePacket(i + m_vert_offset, m_horiz_offset, p); + } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE LinearMapper getLinearMapper(Index i, Index j) const { + if (UseDirectOffsets) { + return LinearMapper(m_base_mapper, i, j); + } + return LinearMapper(m_base_mapper, i + m_vert_offset, j + m_horiz_offset); + } + + template <typename PacketT, int AlignmentType> + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE PacketT load(Index i) const { + EIGEN_STATIC_ASSERT((internal::is_same<PacketT, Packet>::value), YOU_MADE_A_PROGRAMMING_MISTAKE); + const int ActualAlignment = (AlignmentType == Aligned) && (Alignment == Aligned) ? Aligned : Unaligned; + if (UseDirectOffsets) { + return m_base_mapper.template loadPacket<ActualAlignment>(i, 0); + } + return m_base_mapper.template loadPacket<ActualAlignment>(i + m_vert_offset, m_horiz_offset); + } + + template <typename Packet> + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool aligned(Index) const { + return false; + } + + private: + ParentMapper m_base_mapper; + const Index m_vert_offset; + const Index m_horiz_offset; +}; + + +template<typename Scalar_, typename Index, int side, + typename Tensor, + typename nocontract_t, typename contract_t, + int packet_size, + bool inner_dim_contiguous, bool inner_dim_reordered, int Alignment> +class TensorContractionInputMapper + : public BaseTensorContractionMapper<Scalar_, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment> { + + public: + typedef Scalar_ Scalar; + typedef BaseTensorContractionMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment> Base; + typedef TensorContractionSubMapper<Scalar, Index, side, Tensor, nocontract_t, contract_t, packet_size, inner_dim_contiguous, inner_dim_reordered, Alignment> SubMapper; + typedef SubMapper VectorMapper; + + EIGEN_DEVICE_FUNC TensorContractionInputMapper(const Tensor& tensor, + const nocontract_t& nocontract_strides, + const nocontract_t& ij_strides, + const contract_t& contract_strides, + const contract_t& k_strides) + : Base(tensor, nocontract_strides, ij_strides, contract_strides, k_strides) { } + + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE SubMapper getSubMapper(Index i, Index j) const { + return SubMapper(*this, i, j); + } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE VectorMapper getVectorMapper(Index i, Index j) const { + return VectorMapper(*this, i, j); + } +}; + + + +} // end namespace internal +} // end namespace Eigen + +#endif // EIGEN_CXX11_TENSOR_TENSOR_CONTRACTION_MAPPER_H diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h index 576bea295..9044454fd 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorContractionThreadPool.h @@ -28,7 +28,7 @@ struct packLhsArg { template<typename LhsScalar, typename RhsScalar, typename RhsMapper, typename OutputMapper, typename Index> struct packRhsAndKernelArg { - const std::vector<LhsScalar*>* blockAs; + const MaxSizeVector<LhsScalar*>* blockAs; RhsScalar* blockB; const RhsMapper& rhs; OutputMapper& output; @@ -46,8 +46,8 @@ struct packRhsAndKernelArg { const Index n_block_idx; const Index m_blocks; const Index n_blocks; - std::vector<Notification*>* kernel_notifications; - const std::vector<Notification*>* lhs_notifications; + MaxSizeVector<Notification*>* kernel_notifications; + const MaxSizeVector<Notification*>* lhs_notifications; const bool need_to_pack; }; @@ -65,10 +65,9 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT typedef TensorContractionOp<Indices, LeftArgType, RightArgType> XprType; typedef typename internal::remove_const<typename XprType::Scalar>::type Scalar; - typedef typename XprType::Packet Packet; typedef typename XprType::Index Index; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; enum { Layout = TensorEvaluator<LeftArgType, Device>::Layout, @@ -136,8 +135,8 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT this->m_device.memset(buffer, 0, m * n * sizeof(Scalar)); - const int lhs_packet_size = internal::packet_traits<LhsScalar>::size; - const int rhs_packet_size = internal::packet_traits<RhsScalar>::size; + const int lhs_packet_size = internal::unpacket_traits<typename LeftEvaluator::PacketReturnType>::size; + const int rhs_packet_size = internal::unpacket_traits<typename RightEvaluator::PacketReturnType>::size; typedef internal::TensorContractionInputMapper<LhsScalar, Index, internal::Lhs, LeftEvaluator, left_nocontract_t, @@ -176,10 +175,10 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT // compute block sizes (which depend on number of threads) const Index num_threads = this->m_device.numThreads(); - Index mc = m; - Index nc = n; - Index kc = k; - internal::computeProductBlockingSizes<LhsScalar,RhsScalar,1>(kc, mc, nc, num_threads); + internal::TensorContractionBlocking<LhsMapper, RhsMapper, Index, internal::ShardByCol> blocking(k, m, n, num_threads); + Index mc = blocking.mc(); + Index nc = blocking.nc(); + Index kc = blocking.kc(); eigen_assert(mc <= m); eigen_assert(nc <= n); eigen_assert(kc <= k); @@ -203,8 +202,7 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT // the alignment requirements with the assumption that // (Traits::mr * sizeof(ResScalar)) % 16 == 0 const Index numBlockAs = numext::mini(num_threads, m_blocks); - std::vector<LhsScalar *> blockAs; - blockAs.reserve(num_threads); + MaxSizeVector<LhsScalar *> blockAs(num_threads); for (int i = 0; i < num_threads; i++) { blockAs.push_back(static_cast<LhsScalar *>(this->m_device.allocate(sizeA * sizeof(LhsScalar)))); } @@ -213,18 +211,17 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT // TODO: is this too much memory to allocate? This simplifies coding a lot, but is wasteful. // Other options: (1) reuse memory when a thread finishes. con: tricky // (2) allocate block B memory in each thread. con: overhead - std::vector<RhsScalar *> blockBs; - blockBs.reserve(n_blocks); + MaxSizeVector<RhsScalar *> blockBs(n_blocks); for (int i = 0; i < n_blocks; i++) { blockBs.push_back(static_cast<RhsScalar *>(this->m_device.allocate(sizeB * sizeof(RhsScalar)))); } // lhs_notifications starts with all null Notifications - std::vector<Notification*> lhs_notifications(num_threads, nullptr); + MaxSizeVector<Notification*> lhs_notifications(num_threads, nullptr); // this should really be numBlockAs * n_blocks; const Index num_kernel_notifications = num_threads * n_blocks; - std::vector<Notification*> kernel_notifications(num_kernel_notifications, + MaxSizeVector<Notification*> kernel_notifications(num_kernel_notifications, nullptr); for (Index k_block_idx = 0; k_block_idx < k_blocks; k_block_idx++) { diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorConversion.h b/unsupported/Eigen/CXX11/src/Tensor/TensorConversion.h index 3ca7daf32..a96776a77 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorConversion.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorConversion.h @@ -25,7 +25,6 @@ struct traits<TensorConversionOp<TargetType, XprType> > { // Type promotion to handle the case where the types of the lhs and the rhs are different. typedef TargetType Scalar; - typedef typename packet_traits<Scalar>::type Packet; typedef typename traits<XprType>::StorageKind StorageKind; typedef typename traits<XprType>::Index Index; typedef typename XprType::Nested Nested; @@ -86,6 +85,27 @@ struct PacketConverter<TensorEvaluator, SrcPacket, TgtPacket, 2, 1> { const TensorEvaluator& m_impl; }; +template <typename TensorEvaluator, typename SrcPacket, typename TgtPacket> +struct PacketConverter<TensorEvaluator, SrcPacket, TgtPacket, 4, 1> { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + PacketConverter(const TensorEvaluator& impl) + : m_impl(impl) {} + + template<int LoadMode, typename Index> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TgtPacket packet(Index index) const { + const int SrcPacketSize = internal::unpacket_traits<SrcPacket>::size; + + SrcPacket src1 = m_impl.template packet<LoadMode>(index); + SrcPacket src2 = m_impl.template packet<LoadMode>(index + SrcPacketSize); + SrcPacket src3 = m_impl.template packet<LoadMode>(index + 2 * SrcPacketSize); + SrcPacket src4 = m_impl.template packet<LoadMode>(index + 3 * SrcPacketSize); + TgtPacket result = internal::pcast<SrcPacket, TgtPacket>(src1, src2, src3, src4); + return result; + } + + private: + const TensorEvaluator& m_impl; +}; template <typename TensorEvaluator, typename SrcPacket, typename TgtPacket> struct PacketConverter<TensorEvaluator, SrcPacket, TgtPacket, 1, 2> { @@ -104,9 +124,12 @@ struct PacketConverter<TensorEvaluator, SrcPacket, TgtPacket, 1, 2> { return internal::pcast<SrcPacket, TgtPacket>(m_impl.template packet<Unaligned>(index)); } else { const int TgtPacketSize = internal::unpacket_traits<TgtPacket>::size; + typedef typename internal::unpacket_traits<SrcPacket>::type SrcType; + typedef typename internal::unpacket_traits<TgtPacket>::type TgtType; + internal::scalar_cast_op<SrcType, TgtType> converter; EIGEN_ALIGN_MAX typename internal::unpacket_traits<TgtPacket>::type values[TgtPacketSize]; for (int i = 0; i < TgtPacketSize; ++i) { - values[i] = m_impl.coeff(index+i); + values[i] = converter(m_impl.coeff(index+i)); } TgtPacket rslt = internal::pload<TgtPacket>(values); return rslt; @@ -123,12 +146,10 @@ class TensorConversionOp : public TensorBase<TensorConversionOp<TargetType, XprT { public: typedef typename internal::traits<TensorConversionOp>::Scalar Scalar; - typedef typename internal::traits<TensorConversionOp>::Packet Packet; typedef typename internal::traits<TensorConversionOp>::StorageKind StorageKind; typedef typename internal::traits<TensorConversionOp>::Index Index; typedef typename internal::nested<TensorConversionOp>::type Nested; typedef Scalar CoeffReturnType; - typedef Packet PacketReturnType; typedef typename NumTraits<Scalar>::Real RealScalar; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorConversionOp(const XprType& xpr) @@ -142,6 +163,18 @@ class TensorConversionOp : public TensorBase<TensorConversionOp<TargetType, XprT typename XprType::Nested m_xpr; }; +template <bool SameType, typename Eval, typename Scalar> struct ConversionSubExprEval { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static bool run(Eval& impl, Scalar*) { + impl.evalSubExprsIfNeeded(NULL); + return true; + } +}; + +template <typename Eval, typename Scalar> struct ConversionSubExprEval<true, Eval, Scalar> { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static bool run(Eval& impl, Scalar* data) { + return impl.evalSubExprsIfNeeded(data); + } +}; @@ -155,13 +188,14 @@ struct TensorEvaluator<const TensorConversionOp<TargetType, ArgType>, Device> typedef TargetType Scalar; typedef TargetType CoeffReturnType; typedef typename internal::remove_all<typename internal::traits<ArgType>::Scalar>::type SrcType; - typedef typename internal::traits<XprType>::Packet PacketReturnType; - typedef typename internal::packet_traits<SrcType>::type PacketSourceType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; + typedef typename PacketType<SrcType, Device>::type PacketSourceType; enum { IsAligned = false, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess && internal::type_casting_traits<SrcType, TargetType>::VectorizedCast, Layout = TensorEvaluator<ArgType, Device>::Layout, + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -171,10 +205,9 @@ struct TensorEvaluator<const TensorConversionOp<TargetType, ArgType>, Device> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_impl.dimensions(); } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* /*data*/) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* data) { - m_impl.evalSubExprsIfNeeded(NULL); - return true; + return ConversionSubExprEval<internal::is_same<TargetType, SrcType>::value, TensorEvaluator<ArgType, Device>, Scalar>::run(m_impl, data); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorConvolution.h b/unsupported/Eigen/CXX11/src/Tensor/TensorConvolution.h index a82bfc0aa..4fe1fb943 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorConvolution.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorConvolution.h @@ -21,7 +21,7 @@ namespace Eigen { */ namespace internal { -template <typename Index, typename InputDims, size_t NumKernelDims, int Layout> +template <typename Index, typename InputDims, int NumKernelDims, int Layout> class IndexMapper { public: IndexMapper(const InputDims& input_dims, const array<Index, NumKernelDims>& kernel_dims, @@ -123,7 +123,7 @@ class IndexMapper { } inputIndex += p * m_inputStrides[NumKernelDims]; } else { - int limit = 0; + std::ptrdiff_t limit = 0; if (NumKernelDims < NumDims) { limit = NumDims - NumKernelDims - 1; } @@ -147,7 +147,7 @@ class IndexMapper { } outputIndex += p * m_outputStrides[NumKernelDims]; } else { - int limit = 0; + std::ptrdiff_t limit = 0; if (NumKernelDims < NumDims) { limit = NumDims - NumKernelDims - 1; } @@ -206,7 +206,7 @@ class IndexMapper { } private: - static const size_t NumDims = internal::array_size<InputDims>::value; + static const int NumDims = internal::array_size<InputDims>::value; array<Index, NumDims> m_inputStrides; array<Index, NumDims> m_outputStrides; array<Index, NumDims> m_cudaInputStrides; @@ -221,7 +221,6 @@ struct traits<TensorConvolutionOp<Dimensions, InputXprType, KernelXprType> > // Type promotion to handle the case where the types of the lhs and the rhs are different. typedef typename promote_storage_type<typename InputXprType::Scalar, typename KernelXprType::Scalar>::ret Scalar; - typedef typename packet_traits<Scalar>::type Packet; typedef typename promote_storage_type<typename traits<InputXprType>::StorageKind, typename traits<KernelXprType>::StorageKind>::ret StorageKind; typedef typename promote_index_type<typename traits<InputXprType>::Index, @@ -259,12 +258,9 @@ class TensorConvolutionOp : public TensorBase<TensorConvolutionOp<Indices, Input { public: typedef typename Eigen::internal::traits<TensorConvolutionOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorConvolutionOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename internal::promote_storage_type<typename InputXprType::CoeffReturnType, typename KernelXprType::CoeffReturnType>::ret CoeffReturnType; - typedef typename internal::promote_storage_type<typename InputXprType::PacketReturnType, - typename KernelXprType::PacketReturnType>::ret PacketReturnType; typedef typename Eigen::internal::nested<TensorConvolutionOp>::type Nested; typedef typename Eigen::internal::traits<TensorConvolutionOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorConvolutionOp>::Index Index; @@ -306,6 +302,7 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr PacketAccess = TensorEvaluator<InputArgType, Device>::PacketAccess & TensorEvaluator<KernelArgType, Device>::PacketAccess, Layout = TensorEvaluator<InputArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -372,7 +369,7 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -752,6 +749,7 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr PacketAccess = false, Layout = TensorEvaluator<InputArgType, GpuDevice>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const GpuDevice& device) @@ -773,7 +771,7 @@ struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelAr } typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, GpuDevice>::type PacketReturnType; typedef typename InputArgType::Scalar Scalar; EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_dimensions; } diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorCustomOp.h b/unsupported/Eigen/CXX11/src/Tensor/TensorCustomOp.h index 0157f6fab..b58e513b4 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorCustomOp.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorCustomOp.h @@ -24,7 +24,6 @@ template<typename CustomUnaryFunc, typename XprType> struct traits<TensorCustomUnaryOp<CustomUnaryFunc, XprType> > { typedef typename XprType::Scalar Scalar; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprType::StorageKind StorageKind; typedef typename XprType::Index Index; typedef typename XprType::Nested Nested; @@ -54,10 +53,8 @@ class TensorCustomUnaryOp : public TensorBase<TensorCustomUnaryOp<CustomUnaryFun { public: typedef typename internal::traits<TensorCustomUnaryOp>::Scalar Scalar; - typedef typename internal::traits<TensorCustomUnaryOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; typedef typename internal::nested<TensorCustomUnaryOp>::type Nested; typedef typename internal::traits<TensorCustomUnaryOp>::StorageKind StorageKind; typedef typename internal::traits<TensorCustomUnaryOp>::Index Index; @@ -95,6 +92,7 @@ struct TensorEvaluator<const TensorCustomUnaryOp<CustomUnaryFunc, XprType>, Devi BlockAccess = false, Layout = TensorEvaluator<XprType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const ArgType& op, const Device& device) @@ -104,7 +102,7 @@ struct TensorEvaluator<const TensorCustomUnaryOp<CustomUnaryFunc, XprType>, Devi } typedef typename internal::remove_const<typename XprType::CoeffReturnType>::type CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -166,11 +164,8 @@ struct traits<TensorCustomBinaryOp<CustomBinaryFunc, LhsXprType, RhsXprType> > { typedef typename internal::promote_storage_type<typename LhsXprType::Scalar, typename RhsXprType::Scalar>::ret Scalar; - typedef typename packet_traits<Scalar>::type Packet; typedef typename internal::promote_storage_type<typename LhsXprType::CoeffReturnType, typename RhsXprType::CoeffReturnType>::ret CoeffReturnType; - typedef typename internal::promote_storage_type<typename LhsXprType::PacketReturnType, - typename RhsXprType::PacketReturnType>::ret PacketReturnType; typedef typename promote_storage_type<typename traits<LhsXprType>::StorageKind, typename traits<RhsXprType>::StorageKind>::ret StorageKind; typedef typename promote_index_type<typename traits<LhsXprType>::Index, @@ -204,10 +199,8 @@ class TensorCustomBinaryOp : public TensorBase<TensorCustomBinaryOp<CustomBinary { public: typedef typename internal::traits<TensorCustomBinaryOp>::Scalar Scalar; - typedef typename internal::traits<TensorCustomBinaryOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename internal::traits<TensorCustomBinaryOp>::CoeffReturnType CoeffReturnType; - typedef typename internal::traits<TensorCustomBinaryOp>::PacketReturnType PacketReturnType; typedef typename internal::nested<TensorCustomBinaryOp>::type Nested; typedef typename internal::traits<TensorCustomBinaryOp>::StorageKind StorageKind; typedef typename internal::traits<TensorCustomBinaryOp>::Index Index; @@ -250,6 +243,7 @@ struct TensorEvaluator<const TensorCustomBinaryOp<CustomBinaryFunc, LhsXprType, BlockAccess = false, Layout = TensorEvaluator<LhsXprType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -259,7 +253,7 @@ struct TensorEvaluator<const TensorCustomBinaryOp<CustomBinaryFunc, LhsXprType, } typedef typename internal::remove_const<typename XprType::CoeffReturnType>::type CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceCuda.h b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceCuda.h index c76d1ee3f..821835cf3 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceCuda.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceCuda.h @@ -10,7 +10,6 @@ #if defined(EIGEN_USE_GPU) && !defined(EIGEN_CXX11_TENSOR_TENSOR_DEVICE_CUDA_H) #define EIGEN_CXX11_TENSOR_TENSOR_DEVICE_CUDA_H - namespace Eigen { // This defines an interface that GPUDevice can take to use @@ -35,12 +34,23 @@ static void initializeDeviceProp() { if (!m_devicePropInitialized) { int num_devices; cudaError_t status = cudaGetDeviceCount(&num_devices); - EIGEN_UNUSED_VARIABLE(status) - assert(status == cudaSuccess); + if (status != cudaSuccess) { + std::cerr << "Failed to get the number of CUDA devices: " + << cudaGetErrorString(status) + << std::endl; + assert(status == cudaSuccess); + } m_deviceProperties = new cudaDeviceProp[num_devices]; for (int i = 0; i < num_devices; ++i) { status = cudaGetDeviceProperties(&m_deviceProperties[i], i); - assert(status == cudaSuccess); + if (status != cudaSuccess) { + std::cerr << "Failed to initialize CUDA device #" + << i + << ": " + << cudaGetErrorString(status) + << std::endl; + assert(status == cudaSuccess); + } } m_devicePropInitialized = true; } @@ -110,10 +120,12 @@ class CudaStreamDevice : public StreamInterface { struct GpuDevice { // The StreamInterface is not owned: the caller is // responsible for its initialization and eventual destruction. - explicit GpuDevice(const StreamInterface* stream) : stream_(stream) { + explicit GpuDevice(const StreamInterface* stream) : stream_(stream), max_blocks_(INT_MAX) { + eigen_assert(stream); + } + explicit GpuDevice(const StreamInterface* stream, int num_blocks) : stream_(stream), max_blocks_(num_blocks) { eigen_assert(stream); } - // TODO(bsteiner): This is an internal API, we should not expose it. EIGEN_STRONG_INLINE const cudaStream_t& stream() const { return stream_->stream(); @@ -199,27 +211,68 @@ struct GpuDevice { EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void synchronize() const { #if defined(__CUDACC__) && !defined(__CUDA_ARCH__) cudaError_t err = cudaStreamSynchronize(stream_->stream()); - EIGEN_UNUSED_VARIABLE(err) - assert(err == cudaSuccess); + if (err != cudaSuccess) { + std::cerr << "Error detected in CUDA stream: " + << cudaGetErrorString(err) + << std::endl; + assert(err == cudaSuccess); + } #else assert(false && "The default device should be used instead to generate kernel code"); #endif } - inline int getNumCudaMultiProcessors() const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int getNumCudaMultiProcessors() const { +#ifndef __CUDA_ARCH__ return stream_->deviceProperties().multiProcessorCount; +#else + eigen_assert(false && "The default device should be used instead to generate kernel code"); + return 0; +#endif } - inline int maxCudaThreadsPerBlock() const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int maxCudaThreadsPerBlock() const { +#ifndef __CUDA_ARCH__ return stream_->deviceProperties().maxThreadsPerBlock; +#else + eigen_assert(false && "The default device should be used instead to generate kernel code"); + return 0; +#endif } - inline int maxCudaThreadsPerMultiProcessor() const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int maxCudaThreadsPerMultiProcessor() const { +#ifndef __CUDA_ARCH__ return stream_->deviceProperties().maxThreadsPerMultiProcessor; +#else + eigen_assert(false && "The default device should be used instead to generate kernel code"); + return 0; +#endif } - inline int sharedMemPerBlock() const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int sharedMemPerBlock() const { +#ifndef __CUDA_ARCH__ return stream_->deviceProperties().sharedMemPerBlock; +#else + eigen_assert(false && "The default device should be used instead to generate kernel code"); + return 0; +#endif } - inline int majorDeviceVersion() const { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int majorDeviceVersion() const { +#ifndef __CUDA_ARCH__ return stream_->deviceProperties().major; +#else + eigen_assert(false && "The default device should be used instead to generate kernel code"); + return 0; +#endif + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int minorDeviceVersion() const { +#ifndef __CUDA_ARCH__ + return stream_->deviceProperties().minor; +#else + eigen_assert(false && "The default device should be used instead to generate kernel code"); + return 0; +#endif + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int maxBlocks() const { + return max_blocks_; } // This function checks if the CUDA runtime recorded an error for the @@ -235,24 +288,33 @@ struct GpuDevice { private: const StreamInterface* stream_; - + int max_blocks_; }; - -#define LAUNCH_CUDA_KERNEL(kernel, gridsize, blocksize, sharedmem, device, ...) \ - (kernel) <<< (gridsize), (blocksize), (sharedmem), (device).stream() >>> (__VA_ARGS__); \ +#ifndef __CUDA_ARCH__ +#define LAUNCH_CUDA_KERNEL(kernel, gridsize, blocksize, sharedmem, device, ...) \ + (kernel) <<< (gridsize), (blocksize), (sharedmem), (device).stream() >>> (__VA_ARGS__); \ assert(cudaGetLastError() == cudaSuccess); +#else +#define LAUNCH_CUDA_KERNEL(kernel, ...) \ + { const auto __attribute__((__unused__)) __makeTheKernelInstantiate = &(kernel); } \ + eigen_assert(false && "Cannot launch a kernel from another kernel" __CUDA_ARCH__); +#endif // FIXME: Should be device and kernel specific. #ifdef __CUDACC__ -static inline void setCudaSharedMemConfig(cudaSharedMemConfig config) { +static EIGEN_DEVICE_FUNC inline void setCudaSharedMemConfig(cudaSharedMemConfig config) { +#ifndef __CUDA_ARCH__ cudaError_t status = cudaDeviceSetSharedMemConfig(config); EIGEN_UNUSED_VARIABLE(status) assert(status == cudaSuccess); +#else + EIGEN_UNUSED_VARIABLE(config) +#endif } #endif } // end namespace Eigen -#endif // EIGEN_CXX11_TENSOR_TENSOR_DEVICE_TYPE_H +#endif // EIGEN_CXX11_TENSOR_TENSOR_DEVICE_CUDA_H diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h index dcbef5b03..cd3dd214b 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorDeviceThreadPool.h @@ -24,36 +24,40 @@ class ThreadPoolInterface { // The implementation of the ThreadPool type ensures that the Schedule method // runs the functions it is provided in FIFO order when the scheduling is done // by a single thread. -class ThreadPool : public ThreadPoolInterface { +// Environment provides a way to create threads and also allows to intercept +// task submission and execution. +template <typename Environment> +class ThreadPoolTempl : public ThreadPoolInterface { public: // Construct a pool that contains "num_threads" threads. - explicit ThreadPool(int num_threads) { + explicit ThreadPoolTempl(int num_threads, Environment env = Environment()) + : env_(env), threads_(num_threads), waiters_(num_threads) { for (int i = 0; i < num_threads; i++) { - threads_.push_back(new std::thread([this]() { WorkerLoop(); })); + threads_.push_back(env.CreateThread([this]() { WorkerLoop(); })); } } // Wait until all scheduled work has finished and then destroy the // set of threads. - ~ThreadPool() - { + ~ThreadPoolTempl() { { // Wait for all work to get done. std::unique_lock<std::mutex> l(mu_); - empty_.wait(l, [this]() { return pending_.empty(); }); + while (!pending_.empty()) { + empty_.wait(l); + } exiting_ = true; // Wakeup all waiters. for (auto w : waiters_) { w->ready = true; - w->work = nullptr; + w->task.f = nullptr; w->cv.notify_one(); } } // Wait for threads to finish. for (auto t : threads_) { - t->join(); delete t; } } @@ -61,14 +65,15 @@ class ThreadPool : public ThreadPoolInterface { // Schedule fn() for execution in the pool of threads. The functions are // executed in the order in which they are scheduled. void Schedule(std::function<void()> fn) { + Task t = env_.CreateTask(std::move(fn)); std::unique_lock<std::mutex> l(mu_); if (waiters_.empty()) { - pending_.push_back(fn); + pending_.push_back(std::move(t)); } else { Waiter* w = waiters_.back(); waiters_.pop_back(); w->ready = true; - w->work = fn; + w->task = std::move(t); w->cv.notify_one(); } } @@ -77,88 +82,153 @@ class ThreadPool : public ThreadPoolInterface { void WorkerLoop() { std::unique_lock<std::mutex> l(mu_); Waiter w; + Task t; while (!exiting_) { - std::function<void()> fn; if (pending_.empty()) { // Wait for work to be assigned to me w.ready = false; waiters_.push_back(&w); - w.cv.wait(l, [&w]() { return w.ready; }); - fn = w.work; - w.work = nullptr; + while (!w.ready) { + w.cv.wait(l); + } + t = w.task; + w.task.f = nullptr; } else { // Pick up pending work - fn = pending_.front(); + t = std::move(pending_.front()); pending_.pop_front(); if (pending_.empty()) { empty_.notify_all(); } } - if (fn) { + if (t.f) { mu_.unlock(); - fn(); + env_.ExecuteTask(t); + t.f = nullptr; mu_.lock(); } } } private: + typedef typename Environment::Task Task; + typedef typename Environment::EnvThread Thread; + struct Waiter { std::condition_variable cv; - std::function<void()> work; + Task task; bool ready; }; + Environment env_; std::mutex mu_; - std::vector<std::thread*> threads_; // All threads - std::vector<Waiter*> waiters_; // Stack of waiting threads. - std::deque<std::function<void()>> pending_; // Queue of pending work - std::condition_variable empty_; // Signaled on pending_.empty() + MaxSizeVector<Thread*> threads_; // All threads + MaxSizeVector<Waiter*> waiters_; // Stack of waiting threads. + std::deque<Task> pending_; // Queue of pending work + std::condition_variable empty_; // Signaled on pending_.empty() bool exiting_ = false; }; +struct StlThreadEnvironment { + struct Task { + std::function<void()> f; + }; -// Notification is an object that allows a user to to wait for another -// thread to signal a notification that an event has occurred. -// -// Multiple threads can wait on the same Notification object. -// but only one caller must call Notify() on the object. -class Notification { + // EnvThread constructor must start the thread, + // destructor must join the thread. + class EnvThread { + public: + EnvThread(std::function<void()> f) : thr_(f) {} + ~EnvThread() { thr_.join(); } + + private: + std::thread thr_; + }; + + EnvThread* CreateThread(std::function<void()> f) { return new EnvThread(f); } + Task CreateTask(std::function<void()> f) { return Task{std::move(f)}; } + void ExecuteTask(const Task& t) { t.f(); } +}; + +typedef ThreadPoolTempl<StlThreadEnvironment> ThreadPool; + + +// Barrier is an object that allows one or more threads to wait until +// Notify has been called a specified number of times. +class Barrier { public: - Notification() : notified_(false) {} - ~Notification() {} + Barrier(unsigned int count) : state_(count << 1), notified_(false) { + eigen_assert(((count << 1) >> 1) == count); + } + ~Barrier() { + eigen_assert((state_>>1) == 0); + } void Notify() { + unsigned int v = state_.fetch_sub(2, std::memory_order_acq_rel) - 2; + if (v != 1) { + eigen_assert(((v + 2) & ~1) != 0); + return; // either count has not dropped to 0, or waiter is not waiting + } std::unique_lock<std::mutex> l(mu_); eigen_assert(!notified_); notified_ = true; cv_.notify_all(); } - void WaitForNotification() { + void Wait() { + unsigned int v = state_.fetch_or(1, std::memory_order_acq_rel); + if ((v >> 1) == 0) return; std::unique_lock<std::mutex> l(mu_); - cv_.wait(l, [this]() { return notified_; } ); + while (!notified_) { + cv_.wait(l); + } } private: std::mutex mu_; std::condition_variable cv_; + std::atomic<unsigned int> state_; // low bit is waiter flag bool notified_; }; + +// Notification is an object that allows a user to to wait for another +// thread to signal a notification that an event has occurred. +// +// Multiple threads can wait on the same Notification object, +// but only one caller must call Notify() on the object. +struct Notification : Barrier { + Notification() : Barrier(1) {}; +}; + + // Runs an arbitrary function and then calls Notify() on the passed in // Notification. -template <typename Function, typename... Args> struct FunctionWrapper +template <typename Function, typename... Args> struct FunctionWrapperWithNotification { static void run(Notification* n, Function f, Args... args) { f(args...); - n->Notify(); + if (n) { + n->Notify(); + } } }; -static EIGEN_STRONG_INLINE void wait_until_ready(Notification* n) { +template <typename Function, typename... Args> struct FunctionWrapperWithBarrier +{ + static void run(Barrier* b, Function f, Args... args) { + f(args...); + if (b) { + b->Notify(); + } + } +}; + +template <typename SyncType> +static EIGEN_STRONG_INLINE void wait_until_ready(SyncType* n) { if (n) { - n->WaitForNotification(); + n->Wait(); } } @@ -203,10 +273,20 @@ struct ThreadPoolDevice { EIGEN_STRONG_INLINE Notification* enqueue(Function&& f, Args&&... args) const { Notification* n = new Notification(); std::function<void()> func = - std::bind(&FunctionWrapper<Function, Args...>::run, n, f, args...); + std::bind(&FunctionWrapperWithNotification<Function, Args...>::run, n, f, args...); pool_->Schedule(func); return n; } + + template <class Function, class... Args> + EIGEN_STRONG_INLINE void enqueue_with_barrier(Barrier* b, + Function&& f, + Args&&... args) const { + std::function<void()> func = std::bind( + &FunctionWrapperWithBarrier<Function, Args...>::run, b, f, args...); + pool_->Schedule(func); + } + template <class Function, class... Args> EIGEN_STRONG_INLINE void enqueueNoNotification(Function&& f, Args&&... args) const { std::function<void()> func = std::bind(f, args...); diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorDimensions.h b/unsupported/Eigen/CXX11/src/Tensor/TensorDimensions.h index f3c9a3148..977dcafb0 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorDimensions.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorDimensions.h @@ -110,14 +110,14 @@ struct Sizes : internal::numeric_list<std::ptrdiff_t, Indices...> { return internal::arg_prod(Indices...); } - Sizes() { } + EIGEN_DEVICE_FUNC Sizes() { } template <typename DenseIndex> - explicit Sizes(const array<DenseIndex, Base::count>& /*indices*/) { + explicit EIGEN_DEVICE_FUNC Sizes(const array<DenseIndex, Base::count>& /*indices*/) { // todo: add assertion } #ifdef EIGEN_HAS_VARIADIC_TEMPLATES - template <typename... DenseIndex> Sizes(DenseIndex...) { } - explicit Sizes(std::initializer_list<std::ptrdiff_t> /*l*/) { + template <typename... DenseIndex> EIGEN_DEVICE_FUNC Sizes(DenseIndex...) { } + explicit EIGEN_DEVICE_FUNC Sizes(std::initializer_list<std::ptrdiff_t> /*l*/) { // todo: add assertion } #endif @@ -285,17 +285,17 @@ struct DSizes : array<DenseIndex, NumDims> { } EIGEN_DEVICE_FUNC explicit DSizes(const array<DenseIndex, NumDims>& a) : Base(a) { } -#ifdef EIGEN_HAS_VARIADIC_TEMPLATES - template<typename... IndexTypes> EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE explicit DSizes(DenseIndex firstDimension, IndexTypes... otherDimensions) { - EIGEN_STATIC_ASSERT(sizeof...(otherDimensions) + 1 == NumDims, YOU_MADE_A_PROGRAMMING_MISTAKE) - (*this) = array<DenseIndex, NumDims>{{firstDimension, otherDimensions...}}; - } -#else EIGEN_DEVICE_FUNC explicit DSizes(const DenseIndex i0) { eigen_assert(NumDims == 1); (*this)[0] = i0; } + +#ifdef EIGEN_HAS_VARIADIC_TEMPLATES + template<typename... IndexTypes> EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE explicit DSizes(DenseIndex firstDimension, DenseIndex secondDimension, IndexTypes... otherDimensions) : Base({{firstDimension, secondDimension, otherDimensions...}}) { + EIGEN_STATIC_ASSERT(sizeof...(otherDimensions) + 2 == NumDims, YOU_MADE_A_PROGRAMMING_MISTAKE) + } +#else EIGEN_DEVICE_FUNC explicit DSizes(const DenseIndex i0, const DenseIndex i1) { eigen_assert(NumDims == 2); (*this)[0] = i0; @@ -405,20 +405,20 @@ template <std::size_t n, std::size_t V1, std::size_t V2, std::size_t V3, std::si template <typename Dims1, typename Dims2, size_t n, size_t m> struct sizes_match_below_dim { - static inline bool run(Dims1&, Dims2&) { + static EIGEN_DEVICE_FUNC inline bool run(Dims1&, Dims2&) { return false; } }; template <typename Dims1, typename Dims2, size_t n> struct sizes_match_below_dim<Dims1, Dims2, n, n> { - static inline bool run(Dims1& dims1, Dims2& dims2) { + static EIGEN_DEVICE_FUNC inline bool run(Dims1& dims1, Dims2& dims2) { return (array_get<n-1>(dims1) == array_get<n-1>(dims2)) & sizes_match_below_dim<Dims1, Dims2, n-1, n-1>::run(dims1, dims2); } }; template <typename Dims1, typename Dims2> struct sizes_match_below_dim<Dims1, Dims2, 0, 0> { - static inline bool run(Dims1&, Dims2&) { + static EIGEN_DEVICE_FUNC inline bool run(Dims1&, Dims2&) { return true; } }; @@ -427,7 +427,7 @@ struct sizes_match_below_dim<Dims1, Dims2, 0, 0> { template <typename Dims1, typename Dims2> -bool dimensions_match(Dims1& dims1, Dims2& dims2) { +EIGEN_DEVICE_FUNC bool dimensions_match(Dims1& dims1, Dims2& dims2) { return internal::sizes_match_below_dim<Dims1, Dims2, internal::array_size<Dims1>::value, internal::array_size<Dims2>::value>::run(dims1, dims2); } diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorEvalTo.h b/unsupported/Eigen/CXX11/src/Tensor/TensorEvalTo.h index ff4373f59..1fb27a65b 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorEvalTo.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorEvalTo.h @@ -26,7 +26,6 @@ struct traits<TensorEvalToOp<XprType> > // Type promotion to handle the case where the types of the lhs and the rhs are different. typedef typename XprType::Scalar Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprTraits::StorageKind StorageKind; typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; @@ -61,10 +60,8 @@ class TensorEvalToOp : public TensorBase<TensorEvalToOp<XprType> > { public: typedef typename Eigen::internal::traits<TensorEvalToOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorEvalToOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename internal::remove_const<typename XprType::CoeffReturnType>::type CoeffReturnType; - typedef typename internal::remove_const<typename XprType::PacketReturnType>::type PacketReturnType; typedef typename Eigen::internal::nested<TensorEvalToOp>::type Nested; typedef typename Eigen::internal::traits<TensorEvalToOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorEvalToOp>::Index Index; @@ -90,7 +87,6 @@ struct TensorEvaluator<const TensorEvalToOp<ArgType>, Device> { typedef TensorEvalToOp<ArgType> XprType; typedef typename ArgType::Scalar Scalar; - typedef typename ArgType::Packet Packet; typedef typename TensorEvaluator<ArgType, Device>::Dimensions Dimensions; enum { @@ -98,6 +94,7 @@ struct TensorEvaluator<const TensorEvalToOp<ArgType>, Device> PacketAccess = true, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = true }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -109,11 +106,12 @@ struct TensorEvaluator<const TensorEvalToOp<ArgType>, Device> typedef typename XprType::Index Index; typedef typename internal::remove_const<typename XprType::CoeffReturnType>::type CoeffReturnType; - typedef typename internal::remove_const<typename XprType::PacketReturnType>::type PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_impl.dimensions(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(CoeffReturnType* scalar) { + EIGEN_UNUSED_VARIABLE(scalar); eigen_assert(scalar == NULL); return m_impl.evalSubExprsIfNeeded(m_buffer); } @@ -135,12 +133,12 @@ struct TensorEvaluator<const TensorEvalToOp<ArgType>, Device> } template<int LoadMode> - EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - return internal::ploadt<Packet, LoadMode>(m_buffer + index); + return internal::ploadt<PacketReturnType, LoadMode>(m_buffer + index); } - EIGEN_DEVICE_FUNC CoeffReturnType* data() const { return NULL; } + EIGEN_DEVICE_FUNC CoeffReturnType* data() const { return m_buffer; } private: TensorEvaluator<ArgType, Device> m_impl; diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorEvaluator.h b/unsupported/Eigen/CXX11/src/Tensor/TensorEvaluator.h index 902f25247..947a8ed88 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorEvaluator.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorEvaluator.h @@ -29,9 +29,8 @@ struct TensorEvaluator { typedef typename Derived::Index Index; typedef typename Derived::Scalar Scalar; - typedef typename Derived::Packet Packet; typedef typename Derived::Scalar CoeffReturnType; - typedef typename Derived::Packet PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef typename Derived::Dimensions Dimensions; // NumDimensions is -1 for variable dim tensors @@ -40,9 +39,10 @@ struct TensorEvaluator enum { IsAligned = Derived::IsAligned, - PacketAccess = Derived::PacketAccess, + PacketAccess = (internal::unpacket_traits<PacketReturnType>::size > 1), Layout = Derived::Layout, CoordAccess = NumCoords > 0, + RawAccess = true }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const Derived& m, const Device& device) @@ -74,13 +74,13 @@ struct TensorEvaluator template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - return internal::ploadt<Packet, LoadMode>(m_data + index); + return internal::ploadt<PacketReturnType, LoadMode>(m_data + index); } template <int StoreMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - void writePacket(Index index, const Packet& x) + void writePacket(Index index, const PacketReturnType& x) { - return internal::pstoret<Scalar, Packet, StoreMode>(m_data + index, x); + return internal::pstoret<Scalar, PacketReturnType, StoreMode>(m_data + index, x); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(const array<DenseIndex, NumCoords>& coords) const { @@ -134,9 +134,8 @@ struct TensorEvaluator<const Derived, Device> { typedef typename Derived::Index Index; typedef typename Derived::Scalar Scalar; - typedef typename Derived::Packet Packet; typedef typename Derived::Scalar CoeffReturnType; - typedef typename Derived::Packet PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef typename Derived::Dimensions Dimensions; // NumDimensions is -1 for variable dim tensors @@ -145,9 +144,10 @@ struct TensorEvaluator<const Derived, Device> enum { IsAligned = Derived::IsAligned, - PacketAccess = Derived::PacketAccess, + PacketAccess = (internal::unpacket_traits<PacketReturnType>::size > 1), Layout = Derived::Layout, CoordAccess = NumCoords > 0, + RawAccess = true }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const Derived& m, const Device& device) @@ -174,7 +174,7 @@ struct TensorEvaluator<const Derived, Device> template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - return internal::ploadt_ro<Packet, LoadMode>(m_data + index); + return internal::ploadt_ro<PacketReturnType, LoadMode>(m_data + index); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(const array<DenseIndex, NumCoords>& coords) const { @@ -207,6 +207,7 @@ struct TensorEvaluator<const TensorCwiseNullaryOp<NullaryOp, ArgType>, Device> PacketAccess = internal::functor_traits<NullaryOp>::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC @@ -217,7 +218,7 @@ struct TensorEvaluator<const TensorCwiseNullaryOp<NullaryOp, ArgType>, Device> typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; typedef typename internal::traits<XprType>::Scalar CoeffReturnType; - typedef typename internal::traits<XprType>::Packet PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef typename TensorEvaluator<ArgType, Device>::Dimensions Dimensions; EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_argImpl.dimensions(); } @@ -233,7 +234,7 @@ struct TensorEvaluator<const TensorCwiseNullaryOp<NullaryOp, ArgType>, Device> template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - return m_functor.template packetOp<Index,PacketReturnType>(index); + return m_functor.template packetOp<Index, PacketReturnType>(index); } EIGEN_DEVICE_FUNC CoeffReturnType* data() const { return NULL; } @@ -257,6 +258,7 @@ struct TensorEvaluator<const TensorCwiseUnaryOp<UnaryOp, ArgType>, Device> PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess & internal::functor_traits<UnaryOp>::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) @@ -267,7 +269,7 @@ struct TensorEvaluator<const TensorCwiseUnaryOp<UnaryOp, ArgType>, Device> typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; typedef typename internal::traits<XprType>::Scalar CoeffReturnType; - typedef typename internal::traits<XprType>::Packet PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef typename TensorEvaluator<ArgType, Device>::Dimensions Dimensions; EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_argImpl.dimensions(); } @@ -312,6 +314,7 @@ struct TensorEvaluator<const TensorCwiseBinaryOp<BinaryOp, LeftArgType, RightArg internal::functor_traits<BinaryOp>::PacketAccess, Layout = TensorEvaluator<LeftArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) @@ -326,7 +329,7 @@ struct TensorEvaluator<const TensorCwiseBinaryOp<BinaryOp, LeftArgType, RightArg typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; typedef typename internal::traits<XprType>::Scalar CoeffReturnType; - typedef typename internal::traits<XprType>::Packet PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef typename TensorEvaluator<LeftArgType, Device>::Dimensions Dimensions; EIGEN_DEVICE_FUNC const Dimensions& dimensions() const @@ -378,6 +381,7 @@ struct TensorEvaluator<const TensorSelectOp<IfArgType, ThenArgType, ElseArgType> internal::packet_traits<Scalar>::HasBlend, Layout = TensorEvaluator<IfArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) @@ -393,7 +397,7 @@ struct TensorEvaluator<const TensorSelectOp<IfArgType, ThenArgType, ElseArgType> typedef typename XprType::Index Index; typedef typename internal::traits<XprType>::Scalar CoeffReturnType; - typedef typename internal::traits<XprType>::Packet PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef typename TensorEvaluator<IfArgType, Device>::Dimensions Dimensions; EIGEN_DEVICE_FUNC const Dimensions& dimensions() const diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorExecutor.h b/unsupported/Eigen/CXX11/src/Tensor/TensorExecutor.h index d93e1de1b..4f4e07aaf 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorExecutor.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorExecutor.h @@ -125,23 +125,18 @@ class TensorExecutor<Expression, ThreadPoolDevice, Vectorizable> int blocksz = std::ceil<int>(static_cast<float>(size)/device.numThreads()) + PacketSize - 1; const Index blocksize = numext::maxi<Index>(PacketSize, (blocksz - (blocksz % PacketSize))); - const Index numblocks = size / blocksize; + const unsigned int numblocks = static_cast<unsigned int>(size / blocksize); - std::vector<Notification*> results; - results.reserve(numblocks); - for (int i = 0; i < numblocks; ++i) { - results.push_back(device.enqueue(&EvalRange<Evaluator, Index, Vectorizable>::run, evaluator, i*blocksize, (i+1)*blocksize)); + Barrier barrier(numblocks); + for (unsigned int i = 0; i < numblocks; ++i) { + device.enqueue_with_barrier(&barrier, &EvalRange<Evaluator, Index, Vectorizable>::run, evaluator, i*blocksize, (i+1)*blocksize); } - if (numblocks * blocksize < size) { + if (static_cast<Index>(numblocks) * blocksize < size) { EvalRange<Evaluator, Index, Vectorizable>::run(evaluator, numblocks * blocksize, size); } - for (int i = 0; i < numblocks; ++i) { - wait_until_ready(results[i]); - delete results[i]; - } - + barrier.Wait(); } evaluator.cleanup(); } @@ -156,14 +151,14 @@ template <typename Expression> class TensorExecutor<Expression, GpuDevice, false> { public: typedef typename Expression::Index Index; - static void run(const Expression& expr, const GpuDevice& device); + static EIGEN_DEVICE_FUNC void run(const Expression& expr, const GpuDevice& device); }; template <typename Expression> class TensorExecutor<Expression, GpuDevice, true> { public: typedef typename Expression::Index Index; - static void run(const Expression& expr, const GpuDevice& device); + static EIGEN_DEVICE_FUNC void run(const Expression& expr, const GpuDevice& device); }; #if defined(__CUDACC__) @@ -213,14 +208,14 @@ EigenMetaKernel_Vectorizable(Evaluator memcopied_eval, Index size) { /*static*/ template <typename Expression> -inline void TensorExecutor<Expression, GpuDevice, false>::run(const Expression& expr, const GpuDevice& device) +EIGEN_DEVICE_FUNC inline void TensorExecutor<Expression, GpuDevice, false>::run(const Expression& expr, const GpuDevice& device) { TensorEvaluator<Expression, GpuDevice> evaluator(expr, device); const bool needs_assign = evaluator.evalSubExprsIfNeeded(NULL); if (needs_assign) { const int block_size = device.maxCudaThreadsPerBlock(); - const int max_blocks = device.getNumCudaMultiProcessors() * device.maxCudaThreadsPerMultiProcessor() / block_size; + const int max_blocks = numext::mini<int>(device.maxBlocks(), device.getNumCudaMultiProcessors() * device.maxCudaThreadsPerMultiProcessor() / block_size); const Index size = array_prod(evaluator.dimensions()); // Create a least one block to ensure we won't crash if we're called with tensors of size 0. const int num_blocks = numext::maxi<int>(numext::mini<int>(max_blocks, (size + block_size - 1) / block_size), 1); @@ -232,14 +227,14 @@ inline void TensorExecutor<Expression, GpuDevice, false>::run(const Expression& /*static*/ template<typename Expression> -inline void TensorExecutor<Expression, GpuDevice, true>::run(const Expression& expr, const GpuDevice& device) +EIGEN_DEVICE_FUNC inline void TensorExecutor<Expression, GpuDevice, true>::run(const Expression& expr, const GpuDevice& device) { TensorEvaluator<Expression, GpuDevice> evaluator(expr, device); const bool needs_assign = evaluator.evalSubExprsIfNeeded(NULL); if (needs_assign) { const int block_size = device.maxCudaThreadsPerBlock(); - const int max_blocks = device.getNumCudaMultiProcessors() * device.maxCudaThreadsPerMultiProcessor() / block_size; + const int max_blocks = numext::mini<int>(device.maxBlocks(), device.getNumCudaMultiProcessors() * device.maxCudaThreadsPerMultiProcessor() / block_size); const Index size = array_prod(evaluator.dimensions()); // Create a least one block to ensure we won't crash if we're called with tensors of size 0. const int num_blocks = numext::maxi<int>(numext::mini<int>(max_blocks, (size + block_size - 1) / block_size), 1); diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorExpr.h b/unsupported/Eigen/CXX11/src/Tensor/TensorExpr.h index 194c68929..49d849e23 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorExpr.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorExpr.h @@ -32,7 +32,6 @@ template<typename NullaryOp, typename XprType> struct traits<TensorCwiseNullaryOp<NullaryOp, XprType> > : traits<XprType> { - typedef typename XprType::Packet Packet; typedef traits<XprType> XprTraits; typedef typename XprType::Scalar Scalar; typedef typename XprType::Nested XprTypeNested; @@ -54,10 +53,8 @@ class TensorCwiseNullaryOp : public TensorBase<TensorCwiseNullaryOp<NullaryOp, X { public: typedef typename Eigen::internal::traits<TensorCwiseNullaryOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorCwiseNullaryOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; typedef TensorCwiseNullaryOp<NullaryOp, XprType> Nested; typedef typename Eigen::internal::traits<TensorCwiseNullaryOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorCwiseNullaryOp>::Index Index; @@ -88,7 +85,6 @@ struct traits<TensorCwiseUnaryOp<UnaryOp, XprType> > // current Scalar/Packet to see if the intent is Input or Output. typedef typename result_of<UnaryOp(typename XprType::Scalar)>::type Scalar; typedef traits<XprType> XprTraits; - typedef typename internal::packet_traits<Scalar>::type Packet; typedef typename XprType::Nested XprTypeNested; typedef typename remove_reference<XprTypeNested>::type _XprTypeNested; static const int NumDimensions = XprTraits::NumDimensions; @@ -118,10 +114,8 @@ class TensorCwiseUnaryOp : public TensorBase<TensorCwiseUnaryOp<UnaryOp, XprType // TODO(phli): Add InputScalar, InputPacket. Check references to // current Scalar/Packet to see if the intent is Input or Output. typedef typename Eigen::internal::traits<TensorCwiseUnaryOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorCwiseUnaryOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef Scalar CoeffReturnType; - typedef typename internal::packet_traits<CoeffReturnType>::type PacketReturnType; typedef typename Eigen::internal::nested<TensorCwiseUnaryOp>::type Nested; typedef typename Eigen::internal::traits<TensorCwiseUnaryOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorCwiseUnaryOp>::Index Index; @@ -155,7 +149,6 @@ struct traits<TensorCwiseBinaryOp<BinaryOp, LhsXprType, RhsXprType> > BinaryOp(typename LhsXprType::Scalar, typename RhsXprType::Scalar)>::type Scalar; typedef traits<LhsXprType> XprTraits; - typedef typename internal::packet_traits<Scalar>::type Packet; typedef typename promote_storage_type< typename traits<LhsXprType>::StorageKind, typename traits<RhsXprType>::StorageKind>::ret StorageKind; @@ -197,10 +190,8 @@ class TensorCwiseBinaryOp : public TensorBase<TensorCwiseBinaryOp<BinaryOp, LhsX // TODO(phli): Add Lhs/RhsScalar, Lhs/RhsPacket. Check references to // current Scalar/Packet to see if the intent is Inputs or Output. typedef typename Eigen::internal::traits<TensorCwiseBinaryOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorCwiseBinaryOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef Scalar CoeffReturnType; - typedef typename internal::packet_traits<CoeffReturnType>::type PacketReturnType; typedef typename Eigen::internal::nested<TensorCwiseBinaryOp>::type Nested; typedef typename Eigen::internal::traits<TensorCwiseBinaryOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorCwiseBinaryOp>::Index Index; @@ -234,7 +225,6 @@ struct traits<TensorSelectOp<IfXprType, ThenXprType, ElseXprType> > { typedef typename traits<ThenXprType>::Scalar Scalar; typedef traits<ThenXprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename promote_storage_type<typename traits<ThenXprType>::StorageKind, typename traits<ElseXprType>::StorageKind>::ret StorageKind; typedef typename promote_index_type<typename traits<ElseXprType>::Index, @@ -266,12 +256,9 @@ class TensorSelectOp : public TensorBase<TensorSelectOp<IfXprType, ThenXprType, { public: typedef typename Eigen::internal::traits<TensorSelectOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorSelectOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename internal::promote_storage_type<typename ThenXprType::CoeffReturnType, typename ElseXprType::CoeffReturnType>::ret CoeffReturnType; - typedef typename internal::promote_storage_type<typename ThenXprType::PacketReturnType, - typename ElseXprType::PacketReturnType>::ret PacketReturnType; typedef typename Eigen::internal::nested<TensorSelectOp>::type Nested; typedef typename Eigen::internal::traits<TensorSelectOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorSelectOp>::Index Index; diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorFFT.h b/unsupported/Eigen/CXX11/src/Tensor/TensorFFT.h index 215a4ebad..d6db45ade 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorFFT.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorFFT.h @@ -10,8 +10,9 @@ #ifndef EIGEN_CXX11_TENSOR_TENSOR_FFT_H #define EIGEN_CXX11_TENSOR_TENSOR_FFT_H -// NVCC fails to compile this code -#if !defined(__CUDACC__) +// This code requires the ability to initialize arrays of constant +// values directly inside a class. +#if __cplusplus >= 201103L || EIGEN_COMP_MSVC >= 1900 namespace Eigen { @@ -135,6 +136,7 @@ struct TensorEvaluator<const TensorFFTOp<FFT, ArgType, FFTResultType, FFTDir>, D BlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) : m_fft(op.fft()), m_impl(op.expression(), device), m_data(NULL), m_device(device) { @@ -205,7 +207,7 @@ struct TensorEvaluator<const TensorFFTOp<FFT, ArgType, FFTResultType, FFTDir>, D } for (size_t i = 0; i < m_fft.size(); ++i) { - int dim = m_fft[i]; + Index dim = m_fft[i]; eigen_assert(dim >= 0 && dim < NumDims); Index line_len = m_dimensions[dim]; eigen_assert(line_len >= 1); @@ -218,19 +220,39 @@ struct TensorEvaluator<const TensorFFTOp<FFT, ArgType, FFTResultType, FFTDir>, D ComplexScalar* b = is_power_of_two ? NULL : (ComplexScalar*)m_device.allocate(sizeof(ComplexScalar) * good_composite); ComplexScalar* pos_j_base_powered = is_power_of_two ? NULL : (ComplexScalar*)m_device.allocate(sizeof(ComplexScalar) * (line_len + 1)); if (!is_power_of_two) { - ComplexScalar pos_j_base = ComplexScalar(std::cos(M_PI/line_len), std::sin(M_PI/line_len)); - for (Index j = 0; j < line_len + 1; ++j) { - pos_j_base_powered[j] = std::pow(pos_j_base, j * j); + // Compute twiddle factors + // t_n = exp(sqrt(-1) * pi * n^2 / line_len) + // for n = 0, 1,..., line_len-1. + // For n > 2 we use the recurrence t_n = t_{n-1}^2 / t_{n-2} * t_1^2 + pos_j_base_powered[0] = ComplexScalar(1, 0); + if (line_len > 1) { + const RealScalar pi_over_len(EIGEN_PI / line_len); + const ComplexScalar pos_j_base = ComplexScalar( + std::cos(pi_over_len), std::sin(pi_over_len)); + pos_j_base_powered[1] = pos_j_base; + if (line_len > 2) { + const ComplexScalar pos_j_base_sq = pos_j_base * pos_j_base; + for (int j = 2; j < line_len + 1; ++j) { + pos_j_base_powered[j] = pos_j_base_powered[j - 1] * + pos_j_base_powered[j - 1] / + pos_j_base_powered[j - 2] * pos_j_base_sq; + } + } } } for (Index partial_index = 0; partial_index < m_size / line_len; ++partial_index) { - Index base_offset = getBaseOffsetFromIndex(partial_index, dim); + const Index base_offset = getBaseOffsetFromIndex(partial_index, dim); // get data into line_buf - for (Index j = 0; j < line_len; ++j) { - Index offset = getIndexFromOffset(base_offset, dim, j); - line_buf[j] = buf[offset]; + const Index stride = m_strides[dim]; + if (stride == 1) { + memcpy(line_buf, &buf[base_offset], line_len*sizeof(ComplexScalar)); + } else { + Index offset = base_offset; + for (int j = 0; j < line_len; ++j, offset += stride) { + line_buf[j] = buf[offset]; + } } // processs the line @@ -242,14 +264,18 @@ struct TensorEvaluator<const TensorFFTOp<FFT, ArgType, FFTResultType, FFTDir>, D } // write back - for (Index j = 0; j < line_len; ++j) { - const ComplexScalar div_factor = (FFTDir == FFT_FORWARD) ? ComplexScalar(1, 0) : ComplexScalar(line_len, 0); - Index offset = getIndexFromOffset(base_offset, dim, j); - buf[offset] = line_buf[j] / div_factor; + if (FFTDir == FFT_FORWARD && stride == 1) { + memcpy(&buf[base_offset], line_buf, line_len*sizeof(ComplexScalar)); + } else { + Index offset = base_offset; + const ComplexScalar div_factor = ComplexScalar(1.0 / line_len, 0); + for (int j = 0; j < line_len; ++j, offset += stride) { + buf[offset] = (FFTDir == FFT_FORWARD) ? line_buf[j] : line_buf[j] * div_factor; + } } } m_device.deallocate(line_buf); - if (!pos_j_base_powered) { + if (!is_power_of_two) { m_device.deallocate(a); m_device.deallocate(b); m_device.deallocate(pos_j_base_powered); @@ -371,109 +397,130 @@ struct TensorEvaluator<const TensorFFTOp<FFT, ArgType, FFTResultType, FFTDir>, D } } - template<int Dir> - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void compute_1D_Butterfly(ComplexScalar* data, Index n, Index n_power_of_2) { - eigen_assert(isPowerOfTwo(n)); - if (n == 1) { - return; + template <int Dir> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void butterfly_2(ComplexScalar* data) { + ComplexScalar tmp = data[1]; + data[1] = data[0] - data[1]; + data[0] += tmp; + } + + template <int Dir> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void butterfly_4(ComplexScalar* data) { + ComplexScalar tmp[4]; + tmp[0] = data[0] + data[1]; + tmp[1] = data[0] - data[1]; + tmp[2] = data[2] + data[3]; + if (Dir == FFT_FORWARD) { + tmp[3] = ComplexScalar(0.0, -1.0) * (data[2] - data[3]); + } else { + tmp[3] = ComplexScalar(0.0, 1.0) * (data[2] - data[3]); } - else if (n == 2) { - ComplexScalar tmp = data[1]; - data[1] = data[0] - data[1]; - data[0] += tmp; - return; + data[0] = tmp[0] + tmp[2]; + data[1] = tmp[1] + tmp[3]; + data[2] = tmp[0] - tmp[2]; + data[3] = tmp[1] - tmp[3]; + } + + template <int Dir> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void butterfly_8(ComplexScalar* data) { + ComplexScalar tmp_1[8]; + ComplexScalar tmp_2[8]; + + tmp_1[0] = data[0] + data[1]; + tmp_1[1] = data[0] - data[1]; + tmp_1[2] = data[2] + data[3]; + if (Dir == FFT_FORWARD) { + tmp_1[3] = (data[2] - data[3]) * ComplexScalar(0, -1); + } else { + tmp_1[3] = (data[2] - data[3]) * ComplexScalar(0, 1); } - else if (n == 4) { - ComplexScalar tmp[4]; - tmp[0] = data[0] + data[1]; - tmp[1] = data[0] - data[1]; - tmp[2] = data[2] + data[3]; - if(Dir == FFT_FORWARD) { - tmp[3] = ComplexScalar(0.0, -1.0) * (data[2] - data[3]); - } - else { - tmp[3] = ComplexScalar(0.0, 1.0) * (data[2] - data[3]); - } - data[0] = tmp[0] + tmp[2]; - data[1] = tmp[1] + tmp[3]; - data[2] = tmp[0] - tmp[2]; - data[3] = tmp[1] - tmp[3]; - return; + tmp_1[4] = data[4] + data[5]; + tmp_1[5] = data[4] - data[5]; + tmp_1[6] = data[6] + data[7]; + if (Dir == FFT_FORWARD) { + tmp_1[7] = (data[6] - data[7]) * ComplexScalar(0, -1); + } else { + tmp_1[7] = (data[6] - data[7]) * ComplexScalar(0, 1); } - else if (n == 8) { - ComplexScalar tmp_1[8]; - ComplexScalar tmp_2[8]; - - tmp_1[0] = data[0] + data[1]; - tmp_1[1] = data[0] - data[1]; - tmp_1[2] = data[2] + data[3]; - if (Dir == FFT_FORWARD) { - tmp_1[3] = (data[2] - data[3]) * ComplexScalar(0, -1); - } - else { - tmp_1[3] = (data[2] - data[3]) * ComplexScalar(0, 1); - } - tmp_1[4] = data[4] + data[5]; - tmp_1[5] = data[4] - data[5]; - tmp_1[6] = data[6] + data[7]; - if (Dir == FFT_FORWARD) { - tmp_1[7] = (data[6] - data[7]) * ComplexScalar(0, -1); - } - else { - tmp_1[7] = (data[6] - data[7]) * ComplexScalar(0, 1); - } - tmp_2[0] = tmp_1[0] + tmp_1[2]; - tmp_2[1] = tmp_1[1] + tmp_1[3]; - tmp_2[2] = tmp_1[0] - tmp_1[2]; - tmp_2[3] = tmp_1[1] - tmp_1[3]; - tmp_2[4] = tmp_1[4] + tmp_1[6]; - // SQRT2DIV2 = sqrt(2)/2 - #define SQRT2DIV2 0.7071067811865476 - if (Dir == FFT_FORWARD) { - tmp_2[5] = (tmp_1[5] + tmp_1[7]) * ComplexScalar(SQRT2DIV2, -SQRT2DIV2); - tmp_2[6] = (tmp_1[4] - tmp_1[6]) * ComplexScalar(0, -1); - tmp_2[7] = (tmp_1[5] - tmp_1[7]) * ComplexScalar(-SQRT2DIV2, -SQRT2DIV2); - } - else { - tmp_2[5] = (tmp_1[5] + tmp_1[7]) * ComplexScalar(SQRT2DIV2, SQRT2DIV2); - tmp_2[6] = (tmp_1[4] - tmp_1[6]) * ComplexScalar(0, 1); - tmp_2[7] = (tmp_1[5] - tmp_1[7]) * ComplexScalar(-SQRT2DIV2, SQRT2DIV2); - } - data[0] = tmp_2[0] + tmp_2[4]; - data[1] = tmp_2[1] + tmp_2[5]; - data[2] = tmp_2[2] + tmp_2[6]; - data[3] = tmp_2[3] + tmp_2[7]; - data[4] = tmp_2[0] - tmp_2[4]; - data[5] = tmp_2[1] - tmp_2[5]; - data[6] = tmp_2[2] - tmp_2[6]; - data[7] = tmp_2[3] - tmp_2[7]; - - return; + tmp_2[0] = tmp_1[0] + tmp_1[2]; + tmp_2[1] = tmp_1[1] + tmp_1[3]; + tmp_2[2] = tmp_1[0] - tmp_1[2]; + tmp_2[3] = tmp_1[1] - tmp_1[3]; + tmp_2[4] = tmp_1[4] + tmp_1[6]; +// SQRT2DIV2 = sqrt(2)/2 +#define SQRT2DIV2 0.7071067811865476 + if (Dir == FFT_FORWARD) { + tmp_2[5] = (tmp_1[5] + tmp_1[7]) * ComplexScalar(SQRT2DIV2, -SQRT2DIV2); + tmp_2[6] = (tmp_1[4] - tmp_1[6]) * ComplexScalar(0, -1); + tmp_2[7] = (tmp_1[5] - tmp_1[7]) * ComplexScalar(-SQRT2DIV2, -SQRT2DIV2); + } else { + tmp_2[5] = (tmp_1[5] + tmp_1[7]) * ComplexScalar(SQRT2DIV2, SQRT2DIV2); + tmp_2[6] = (tmp_1[4] - tmp_1[6]) * ComplexScalar(0, 1); + tmp_2[7] = (tmp_1[5] - tmp_1[7]) * ComplexScalar(-SQRT2DIV2, SQRT2DIV2); } - else { - compute_1D_Butterfly<Dir>(data, n/2, n_power_of_2 - 1); - compute_1D_Butterfly<Dir>(data + n/2, n/2, n_power_of_2 - 1); - //Original code: - //RealScalar wtemp = std::sin(M_PI/n); - //RealScalar wpi = -std::sin(2 * M_PI/n); - RealScalar wtemp = m_sin_PI_div_n_LUT[n_power_of_2]; - RealScalar wpi; - if (Dir == FFT_FORWARD) { - wpi = m_minus_sin_2_PI_div_n_LUT[n_power_of_2]; - } - else { - wpi = 0 - m_minus_sin_2_PI_div_n_LUT[n_power_of_2]; - } + data[0] = tmp_2[0] + tmp_2[4]; + data[1] = tmp_2[1] + tmp_2[5]; + data[2] = tmp_2[2] + tmp_2[6]; + data[3] = tmp_2[3] + tmp_2[7]; + data[4] = tmp_2[0] - tmp_2[4]; + data[5] = tmp_2[1] - tmp_2[5]; + data[6] = tmp_2[2] - tmp_2[6]; + data[7] = tmp_2[3] - tmp_2[7]; + } - const ComplexScalar wp(wtemp, wpi); - ComplexScalar w(1.0, 0.0); - for(Index i = 0; i < n/2; i++) { - ComplexScalar temp(data[i + n/2] * w); - data[i + n/2] = data[i] - temp; - data[i] += temp; - w += w * wp; - } - return; + template <int Dir> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void butterfly_1D_merge( + ComplexScalar* data, Index n, Index n_power_of_2) { + // Original code: + // RealScalar wtemp = std::sin(M_PI/n); + // RealScalar wpi = -std::sin(2 * M_PI/n); + const RealScalar wtemp = m_sin_PI_div_n_LUT[n_power_of_2]; + const RealScalar wpi = (Dir == FFT_FORWARD) + ? m_minus_sin_2_PI_div_n_LUT[n_power_of_2] + : -m_minus_sin_2_PI_div_n_LUT[n_power_of_2]; + + const ComplexScalar wp(wtemp, wpi); + const ComplexScalar wp_one = wp + ComplexScalar(1, 0); + const ComplexScalar wp_one_2 = wp_one * wp_one; + const ComplexScalar wp_one_3 = wp_one_2 * wp_one; + const ComplexScalar wp_one_4 = wp_one_3 * wp_one; + const Index n2 = n / 2; + ComplexScalar w(1.0, 0.0); + for (Index i = 0; i < n2; i += 4) { + ComplexScalar temp0(data[i + n2] * w); + ComplexScalar temp1(data[i + 1 + n2] * w * wp_one); + ComplexScalar temp2(data[i + 2 + n2] * w * wp_one_2); + ComplexScalar temp3(data[i + 3 + n2] * w * wp_one_3); + w = w * wp_one_4; + + data[i + n2] = data[i] - temp0; + data[i] += temp0; + + data[i + 1 + n2] = data[i + 1] - temp1; + data[i + 1] += temp1; + + data[i + 2 + n2] = data[i + 2] - temp2; + data[i + 2] += temp2; + + data[i + 3 + n2] = data[i + 3] - temp3; + data[i + 3] += temp3; + } + } + + template <int Dir> + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void compute_1D_Butterfly( + ComplexScalar* data, Index n, Index n_power_of_2) { + eigen_assert(isPowerOfTwo(n)); + if (n > 8) { + compute_1D_Butterfly<Dir>(data, n / 2, n_power_of_2 - 1); + compute_1D_Butterfly<Dir>(data + n / 2, n / 2, n_power_of_2 - 1); + butterfly_1D_merge<Dir>(data, n, n_power_of_2); + } else if (n == 8) { + butterfly_8<Dir>(data); + } else if (n == 4) { + butterfly_4<Dir>(data); + } else if (n == 2) { + butterfly_2<Dir>(data); } } @@ -518,81 +565,81 @@ struct TensorEvaluator<const TensorFFTOp<FFT, ArgType, FFTResultType, FFTDir>, D // This will support a maximum FFT size of 2^32 for each dimension // m_sin_PI_div_n_LUT[i] = (-2) * std::sin(M_PI / std::pow(2,i)) ^ 2; - RealScalar m_sin_PI_div_n_LUT[32] = { - 0.0, - -2, - -0.999999999999999, - -0.292893218813453, - -0.0761204674887130, - -0.0192147195967696, - -0.00481527332780311, - -0.00120454379482761, - -3.01181303795779e-04, - -7.52981608554592e-05, - -1.88247173988574e-05, - -4.70619042382852e-06, - -1.17654829809007e-06, - -2.94137117780840e-07, - -7.35342821488550e-08, - -1.83835707061916e-08, - -4.59589268710903e-09, - -1.14897317243732e-09, - -2.87243293150586e-10, - -7.18108232902250e-11, - -1.79527058227174e-11, - -4.48817645568941e-12, - -1.12204411392298e-12, - -2.80511028480785e-13, - -7.01277571201985e-14, - -1.75319392800498e-14, - -4.38298482001247e-15, - -1.09574620500312e-15, - -2.73936551250781e-16, - -6.84841378126949e-17, - -1.71210344531737e-17, - -4.28025861329343e-18 + const RealScalar m_sin_PI_div_n_LUT[32] = { + RealScalar(0.0), + RealScalar(-2), + RealScalar(-0.999999999999999), + RealScalar(-0.292893218813453), + RealScalar(-0.0761204674887130), + RealScalar(-0.0192147195967696), + RealScalar(-0.00481527332780311), + RealScalar(-0.00120454379482761), + RealScalar(-3.01181303795779e-04), + RealScalar(-7.52981608554592e-05), + RealScalar(-1.88247173988574e-05), + RealScalar(-4.70619042382852e-06), + RealScalar(-1.17654829809007e-06), + RealScalar(-2.94137117780840e-07), + RealScalar(-7.35342821488550e-08), + RealScalar(-1.83835707061916e-08), + RealScalar(-4.59589268710903e-09), + RealScalar(-1.14897317243732e-09), + RealScalar(-2.87243293150586e-10), + RealScalar( -7.18108232902250e-11), + RealScalar(-1.79527058227174e-11), + RealScalar(-4.48817645568941e-12), + RealScalar(-1.12204411392298e-12), + RealScalar(-2.80511028480785e-13), + RealScalar(-7.01277571201985e-14), + RealScalar(-1.75319392800498e-14), + RealScalar(-4.38298482001247e-15), + RealScalar(-1.09574620500312e-15), + RealScalar(-2.73936551250781e-16), + RealScalar(-6.84841378126949e-17), + RealScalar(-1.71210344531737e-17), + RealScalar(-4.28025861329343e-18) }; // m_minus_sin_2_PI_div_n_LUT[i] = -std::sin(2 * M_PI / std::pow(2,i)); - RealScalar m_minus_sin_2_PI_div_n_LUT[32] = { - 0.0, - 0.0, - -1.00000000000000e+00, - -7.07106781186547e-01, - -3.82683432365090e-01, - -1.95090322016128e-01, - -9.80171403295606e-02, - -4.90676743274180e-02, - -2.45412285229123e-02, - -1.22715382857199e-02, - -6.13588464915448e-03, - -3.06795676296598e-03, - -1.53398018628477e-03, - -7.66990318742704e-04, - -3.83495187571396e-04, - -1.91747597310703e-04, - -9.58737990959773e-05, - -4.79368996030669e-05, - -2.39684498084182e-05, - -1.19842249050697e-05, - -5.99211245264243e-06, - -2.99605622633466e-06, - -1.49802811316901e-06, - -7.49014056584716e-07, - -3.74507028292384e-07, - -1.87253514146195e-07, - -9.36267570730981e-08, - -4.68133785365491e-08, - -2.34066892682746e-08, - -1.17033446341373e-08, - -5.85167231706864e-09, - -2.92583615853432e-09 + const RealScalar m_minus_sin_2_PI_div_n_LUT[32] = { + RealScalar(0.0), + RealScalar(0.0), + RealScalar(-1.00000000000000e+00), + RealScalar(-7.07106781186547e-01), + RealScalar(-3.82683432365090e-01), + RealScalar(-1.95090322016128e-01), + RealScalar(-9.80171403295606e-02), + RealScalar(-4.90676743274180e-02), + RealScalar(-2.45412285229123e-02), + RealScalar(-1.22715382857199e-02), + RealScalar(-6.13588464915448e-03), + RealScalar(-3.06795676296598e-03), + RealScalar(-1.53398018628477e-03), + RealScalar(-7.66990318742704e-04), + RealScalar(-3.83495187571396e-04), + RealScalar(-1.91747597310703e-04), + RealScalar(-9.58737990959773e-05), + RealScalar(-4.79368996030669e-05), + RealScalar(-2.39684498084182e-05), + RealScalar(-1.19842249050697e-05), + RealScalar(-5.99211245264243e-06), + RealScalar(-2.99605622633466e-06), + RealScalar(-1.49802811316901e-06), + RealScalar(-7.49014056584716e-07), + RealScalar(-3.74507028292384e-07), + RealScalar(-1.87253514146195e-07), + RealScalar(-9.36267570730981e-08), + RealScalar(-4.68133785365491e-08), + RealScalar(-2.34066892682746e-08), + RealScalar(-1.17033446341373e-08), + RealScalar(-5.85167231706864e-09), + RealScalar(-2.92583615853432e-09) }; }; } // end namespace Eigen -#endif // __CUDACC__ +#endif // EIGEN_HAS_CONSTEXPR #endif // EIGEN_CXX11_TENSOR_TENSOR_FFT_H diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorFixedSize.h b/unsupported/Eigen/CXX11/src/Tensor/TensorFixedSize.h index bf930f6b8..9c0ed43b7 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorFixedSize.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorFixedSize.h @@ -33,7 +33,6 @@ class TensorFixedSize : public TensorBase<TensorFixedSize<Scalar_, Dimensions_, typedef typename internal::traits<Self>::StorageKind StorageKind; typedef typename internal::traits<Self>::Index Index; typedef Scalar_ Scalar; - typedef typename internal::packet_traits<Scalar>::type Packet; typedef typename NumTraits<Scalar>::Real RealScalar; typedef typename Base::CoeffReturnType CoeffReturnType; @@ -41,10 +40,10 @@ class TensorFixedSize : public TensorBase<TensorFixedSize<Scalar_, Dimensions_, enum { IsAligned = bool(EIGEN_MAX_ALIGN_BYTES>0), - PacketAccess = (internal::packet_traits<Scalar>::size > 1), Layout = Options_ & RowMajor ? RowMajor : ColMajor, CoordAccess = true, - }; + RawAccess = true + }; typedef Dimensions_ Dimensions; static const std::size_t NumIndices = Dimensions::count; @@ -53,7 +52,7 @@ class TensorFixedSize : public TensorBase<TensorFixedSize<Scalar_, Dimensions_, TensorStorage<Scalar, Dimensions, Options> m_storage; public: - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index rank() const { return NumIndices; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index rank() const { return NumIndices; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index dimension(std::size_t n) const { return m_storage.dimensions()[n]; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_storage.dimensions(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index size() const { return m_storage.size(); } @@ -68,7 +67,7 @@ class TensorFixedSize : public TensorBase<TensorFixedSize<Scalar_, Dimensions_, #ifdef EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> - inline const Scalar& coeff(Index firstIndex, IndexTypes... otherIndices) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeff(Index firstIndex, IndexTypes... otherIndices) const { // The number of indices used to access a tensor coefficient must be equal to the rank of the tensor. EIGEN_STATIC_ASSERT(sizeof...(otherIndices) + 1 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) @@ -100,7 +99,7 @@ class TensorFixedSize : public TensorBase<TensorFixedSize<Scalar_, Dimensions_, #ifdef EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> - inline Scalar& coeffRef(Index firstIndex, IndexTypes... otherIndices) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index firstIndex, IndexTypes... otherIndices) { // The number of indices used to access a tensor coefficient must be equal to the rank of the tensor. EIGEN_STATIC_ASSERT(sizeof...(otherIndices) + 1 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) @@ -132,7 +131,7 @@ class TensorFixedSize : public TensorBase<TensorFixedSize<Scalar_, Dimensions_, #ifdef EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> - inline const Scalar& operator()(Index firstIndex, IndexTypes... otherIndices) const + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& operator()(Index firstIndex, IndexTypes... otherIndices) const { // The number of indices used to access a tensor coefficient must be equal to the rank of the tensor. EIGEN_STATIC_ASSERT(sizeof...(otherIndices) + 1 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) @@ -171,7 +170,7 @@ class TensorFixedSize : public TensorBase<TensorFixedSize<Scalar_, Dimensions_, #ifdef EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> - inline Scalar& operator()(Index firstIndex, IndexTypes... otherIndices) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& operator()(Index firstIndex, IndexTypes... otherIndices) { // The number of indices used to access a tensor coefficient must be equal to the rank of the tensor. EIGEN_STATIC_ASSERT(sizeof...(otherIndices) + 1 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) @@ -221,7 +220,7 @@ class TensorFixedSize : public TensorBase<TensorFixedSize<Scalar_, Dimensions_, } #ifdef EIGEN_HAVE_RVALUE_REFERENCES - inline TensorFixedSize(Self&& other) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorFixedSize(Self&& other) : m_storage(other.m_storage) { } diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorForcedEval.h b/unsupported/Eigen/CXX11/src/Tensor/TensorForcedEval.h index 65fd25a2e..14f480901 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorForcedEval.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorForcedEval.h @@ -26,7 +26,6 @@ struct traits<TensorForcedEvalOp<XprType> > // Type promotion to handle the case where the types of the lhs and the rhs are different. typedef typename XprType::Scalar Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename traits<XprType>::StorageKind StorageKind; typedef typename traits<XprType>::Index Index; typedef typename XprType::Nested Nested; @@ -60,10 +59,8 @@ class TensorForcedEvalOp : public TensorBase<TensorForcedEvalOp<XprType> > { public: typedef typename Eigen::internal::traits<TensorForcedEvalOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorForcedEvalOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename internal::remove_const<typename XprType::CoeffReturnType>::type CoeffReturnType; - typedef typename internal::remove_const<typename XprType::PacketReturnType>::type PacketReturnType; typedef typename Eigen::internal::nested<TensorForcedEvalOp>::type Nested; typedef typename Eigen::internal::traits<TensorForcedEvalOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorForcedEvalOp>::Index Index; @@ -85,13 +82,13 @@ struct TensorEvaluator<const TensorForcedEvalOp<ArgType>, Device> { typedef TensorForcedEvalOp<ArgType> XprType; typedef typename ArgType::Scalar Scalar; - typedef typename ArgType::Packet Packet; typedef typename TensorEvaluator<ArgType, Device>::Dimensions Dimensions; enum { IsAligned = true, PacketAccess = (internal::packet_traits<Scalar>::size > 1), Layout = TensorEvaluator<ArgType, Device>::Layout, + RawAccess = true }; EIGEN_DEVICE_FUNC TensorEvaluator(const XprType& op, const Device& device) @@ -100,12 +97,11 @@ struct TensorEvaluator<const TensorForcedEvalOp<ArgType>, Device> typedef typename XprType::Index Index; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_impl.dimensions(); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(CoeffReturnType*) { - m_impl.evalSubExprsIfNeeded(NULL); const Index numValues = m_impl.dimensions().TotalSize(); m_buffer = (CoeffReturnType*)m_device.allocate(numValues * sizeof(CoeffReturnType)); // Should initialize the memory in case we're dealing with non POD types. @@ -116,9 +112,8 @@ struct TensorEvaluator<const TensorForcedEvalOp<ArgType>, Device> } typedef TensorEvalToOp<const ArgType> EvalTo; EvalTo evalToTmp(m_buffer, m_op); - const bool PacketAccess = internal::IsVectorizable<Device, ArgType>::value; + const bool PacketAccess = internal::IsVectorizable<Device, const ArgType>::value; internal::TensorExecutor<const EvalTo, Device, PacketAccess>::run(evalToTmp, m_device); - m_impl.cleanup(); return true; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void cleanup() { @@ -134,7 +129,7 @@ struct TensorEvaluator<const TensorForcedEvalOp<ArgType>, Device> template<int LoadMode> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { - return internal::ploadt<Packet, LoadMode>(m_buffer + index); + return internal::ploadt<PacketReturnType, LoadMode>(m_buffer + index); } EIGEN_DEVICE_FUNC Scalar* data() const { return m_buffer; } diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorFunctors.h b/unsupported/Eigen/CXX11/src/Tensor/TensorFunctors.h index 34ba4e392..b7c13f67f 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorFunctors.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorFunctors.h @@ -25,7 +25,34 @@ struct scalar_mod_op { }; template <typename Scalar> struct functor_traits<scalar_mod_op<Scalar> > -{ enum { Cost = 2 * NumTraits<Scalar>::MulCost, PacketAccess = false }; }; +{ enum { Cost = NumTraits<Scalar>::template Div<false>::Cost, PacketAccess = false }; }; + + +/** \internal + * \brief Template functor to compute the modulo between 2 arrays. + */ +template <typename Scalar> +struct scalar_mod2_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_mod2_op); + EIGEN_DEVICE_FUNC inline Scalar operator() (const Scalar& a, const Scalar& b) const { return a % b; } +}; +template <typename Scalar> +struct functor_traits<scalar_mod2_op<Scalar> > +{ enum { Cost = NumTraits<Scalar>::template Div<false>::Cost, PacketAccess = false }; }; + +template <typename Scalar> +struct scalar_fmod_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_fmod_op); + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar + operator()(const Scalar& a, const Scalar& b) const { + return numext::fmod(a, b); + } +}; +template <typename Scalar> +struct functor_traits<scalar_fmod_op<Scalar> > { + enum { Cost = 13, // Reciprocal throughput of FPREM on Haswell. + PacketAccess = false }; +}; /** \internal @@ -72,11 +99,12 @@ template <typename T> struct SumReducer } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T initialize() const { - return static_cast<T>(0); + internal::scalar_cast_op<int, T> conv; + return conv(0); } template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet initializePacket() const { - return pset1<Packet>(0); + return pset1<Packet>(initialize()); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T finalize(const T accum) const { return accum; @@ -93,7 +121,7 @@ template <typename T> struct SumReducer template <typename T> struct MeanReducer { - static const bool PacketAccess = true; + static const bool PacketAccess = !NumTraits<T>::IsInteger; static const bool IsStateful = true; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE @@ -110,11 +138,12 @@ template <typename T> struct MeanReducer } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T initialize() const { - return static_cast<T>(0); + internal::scalar_cast_op<int, T> conv; + return conv(0); } template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet initializePacket() const { - return pset1<Packet>(0); + return pset1<Packet>(initialize()); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T finalize(const T accum) const { return accum / scalarCount_; @@ -147,11 +176,11 @@ template <typename T> struct MaxReducer } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T initialize() const { - return -(std::numeric_limits<T>::max)(); + return Eigen::NumTraits<T>::lowest(); } template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet initializePacket() const { - return pset1<Packet>(-(std::numeric_limits<T>::max)()); + return pset1<Packet>(initialize()); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T finalize(const T accum) const { return accum; @@ -180,11 +209,11 @@ template <typename T> struct MinReducer } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T initialize() const { - return (std::numeric_limits<T>::max)(); + return Eigen::NumTraits<T>::highest(); } template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet initializePacket() const { - return pset1<Packet>((std::numeric_limits<T>::max)()); + return pset1<Packet>(initialize()); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T finalize(const T accum) const { return accum; @@ -214,11 +243,12 @@ template <typename T> struct ProdReducer } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T initialize() const { - return static_cast<T>(1); + internal::scalar_cast_op<int, T> conv; + return conv(1); } template <typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet initializePacket() const { - return pset1<Packet>(1); + return pset1<Packet>(initialize()); } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T finalize(const T accum) const { return accum; @@ -237,6 +267,8 @@ template <typename T> struct ProdReducer struct AndReducer { static const bool PacketAccess = false; + static const bool IsStateful = false; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(bool t, bool* accum) const { *accum = *accum && t; } @@ -250,6 +282,8 @@ struct AndReducer struct OrReducer { static const bool PacketAccess = false; + static const bool IsStateful = false; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reduce(bool t, bool* accum) const { *accum = *accum || t; } @@ -335,50 +369,54 @@ template <typename T> class UniformRandomGenerator { } template<typename Index> - T operator()(Index, Index = 0) const { + T operator()(Index) const { return random<T>(); } - template<typename Index> - typename internal::packet_traits<T>::type packetOp(Index, Index = 0) const { - const int packetSize = internal::packet_traits<T>::size; + template<typename Index, typename PacketType> + PacketType packetOp(Index) const { + const int packetSize = internal::unpacket_traits<PacketType>::size; EIGEN_ALIGN_MAX T values[packetSize]; for (int i = 0; i < packetSize; ++i) { values[i] = random<T>(); } - return internal::pload<typename internal::packet_traits<T>::type>(values); + return internal::pload<PacketType>(values); } private: bool m_deterministic; }; -#if __cplusplus > 199711 +#if __cplusplus > 199711 || EIGEN_COMP_MSVC >= 1900 template <> class UniformRandomGenerator<float> { public: static const bool PacketAccess = true; - UniformRandomGenerator(bool deterministic = true) : m_deterministic(deterministic) { + UniformRandomGenerator(bool deterministic = true) : m_deterministic(deterministic), m_generator(new std::mt19937()) { if (!deterministic) { - m_generator.seed(get_random_seed()); + m_generator->seed(get_random_seed()); } } UniformRandomGenerator(const UniformRandomGenerator<float>& other) { - m_generator.seed(other(0, 0) * UINT_MAX); + m_generator = new std::mt19937(); + m_generator->seed(other(0) * UINT_MAX); m_deterministic = other.m_deterministic; } + ~UniformRandomGenerator() { + delete m_generator; + } template<typename Index> - float operator()(Index, Index = 0) const { - return m_distribution(m_generator); + float operator()(Index) const { + return m_distribution(*m_generator); } - template<typename Index> - typename internal::packet_traits<float>::type packetOp(Index i, Index j = 0) const { - const int packetSize = internal::packet_traits<float>::size; + template<typename Index, typename PacketType> + PacketType packetOp(Index i) const { + const int packetSize = internal::unpacket_traits<PacketType>::size; EIGEN_ALIGN_MAX float values[packetSize]; for (int k = 0; k < packetSize; ++k) { - values[k] = this->operator()(i, j); + values[k] = this->operator()(i); } - return internal::pload<typename internal::packet_traits<float>::type>(values); + return internal::pload<PacketType>(values); } private: @@ -386,7 +424,7 @@ template <> class UniformRandomGenerator<float> { // Make sure m_deterministic comes first to match the layout of the cpu // version of the code. bool m_deterministic; - mutable std::mt19937 m_generator; + std::mt19937* m_generator; mutable std::uniform_real_distribution<float> m_distribution; }; @@ -394,28 +432,32 @@ template <> class UniformRandomGenerator<double> { public: static const bool PacketAccess = true; - UniformRandomGenerator(bool deterministic = true) : m_deterministic(deterministic) { + UniformRandomGenerator(bool deterministic = true) : m_deterministic(deterministic), m_generator(new std::mt19937()) { if (!deterministic) { - m_generator.seed(get_random_seed()); + m_generator->seed(get_random_seed()); } } UniformRandomGenerator(const UniformRandomGenerator<double>& other) { - m_generator.seed(other(0, 0) * UINT_MAX); + m_generator = new std::mt19937(); + m_generator->seed(other(0) * UINT_MAX); m_deterministic = other.m_deterministic; } + ~UniformRandomGenerator() { + delete m_generator; + } template<typename Index> - double operator()(Index, Index = 0) const { - return m_distribution(m_generator); + double operator()(Index) const { + return m_distribution(*m_generator); } - template<typename Index> - typename internal::packet_traits<double>::type packetOp(Index i, Index j = 0) const { - const int packetSize = internal::packet_traits<double>::size; + template<typename Index, typename PacketType> + PacketType packetOp(Index i) const { + const int packetSize = internal::unpacket_traits<PacketType>::size; EIGEN_ALIGN_MAX double values[packetSize]; for (int k = 0; k < packetSize; ++k) { - values[k] = this->operator()(i, j); + values[k] = this->operator()(i); } - return internal::pload<typename internal::packet_traits<double>::type>(values); + return internal::pload<PacketType>(values); } private: @@ -423,7 +465,7 @@ template <> class UniformRandomGenerator<double> { // Make sure m_deterministic comes first to match the layout of the cpu // version of the code. bool m_deterministic; - mutable std::mt19937 m_generator; + std::mt19937* m_generator; mutable std::uniform_real_distribution<double> m_distribution; }; #endif @@ -451,11 +493,12 @@ template <> class UniformRandomGenerator<float> { } template<typename Index> - __device__ float operator()(Index, Index = 0) const { + __device__ float operator()(Index) const { return curand_uniform(&m_state); } - template<typename Index> - __device__ float4 packetOp(Index, Index = 0) const { + template<typename Index, typename PacketType> + __device__ float4 packetOp(Index) const { + EIGEN_STATIC_ASSERT((is_same<PacketType, float4>::value), YOU_MADE_A_PROGRAMMING_MISTAKE); return curand_uniform4(&m_state); } @@ -480,11 +523,12 @@ template <> class UniformRandomGenerator<double> { curand_init(seed, tid, 0, &m_state); } template<typename Index> - __device__ double operator()(Index, Index = 0) const { + __device__ double operator()(Index) const { return curand_uniform_double(&m_state); } - template<typename Index> - __device__ double2 packetOp(Index, Index = 0) const { + template<typename Index, typename PacketType> + __device__ double2 packetOp(Index) const { + EIGEN_STATIC_ASSERT((is_same<PacketType, double2>::value), YOU_MADE_A_PROGRAMMING_MISTAKE); return curand_uniform2_double(&m_state); } @@ -509,7 +553,7 @@ template <> class UniformRandomGenerator<std::complex<float> > { curand_init(seed, tid, 0, &m_state); } template<typename Index> - __device__ std::complex<float> operator()(Index, Index = 0) const { + __device__ std::complex<float> operator()(Index) const { float4 vals = curand_uniform4(&m_state); return std::complex<float>(vals.x, vals.y); } @@ -535,7 +579,7 @@ template <> class UniformRandomGenerator<std::complex<double> > { curand_init(seed, tid, 0, &m_state); } template<typename Index> - __device__ std::complex<double> operator()(Index, Index = 0) const { + __device__ std::complex<double> operator()(Index) const { double2 vals = curand_uniform2_double(&m_state); return std::complex<double>(vals.x, vals.y); } @@ -547,41 +591,54 @@ template <> class UniformRandomGenerator<std::complex<double> > { #endif +template <typename Scalar> +struct functor_traits<UniformRandomGenerator<Scalar> > { + enum { + PacketAccess = UniformRandomGenerator<Scalar>::PacketAccess + }; +}; + -#if (!defined (EIGEN_USE_GPU) || !defined(__CUDACC__) || !defined(__CUDA_ARCH__)) && __cplusplus > 199711 + +#if (!defined (EIGEN_USE_GPU) || !defined(__CUDACC__) || !defined(__CUDA_ARCH__)) && (__cplusplus > 199711 || EIGEN_COMP_MSVC >= 1900) // We're not compiling a cuda kernel template <typename T> class NormalRandomGenerator { public: static const bool PacketAccess = true; - NormalRandomGenerator(bool deterministic = true) : m_deterministic(deterministic), m_distribution(0, 1) { + NormalRandomGenerator(bool deterministic = true) : m_deterministic(deterministic), m_distribution(0, 1), m_generator(new std::mt19937()) { if (!deterministic) { - m_generator.seed(get_random_seed()); + m_generator->seed(get_random_seed()); } } NormalRandomGenerator(const NormalRandomGenerator& other) - : m_deterministic(other.m_deterministic), m_distribution(other.m_distribution) { - m_generator.seed(other(0, 0) * UINT_MAX); + : m_deterministic(other.m_deterministic), m_distribution(other.m_distribution), m_generator(new std::mt19937()) { + m_generator->seed(other(0) * UINT_MAX); } - - template<typename Index> - T operator()(Index, Index = 0) const { - return m_distribution(m_generator); + ~NormalRandomGenerator() { + delete m_generator; } template<typename Index> - typename internal::packet_traits<T>::type packetOp(Index, Index = 0) const { - const int packetSize = internal::packet_traits<T>::size; + T operator()(Index) const { + return m_distribution(*m_generator); + } + template<typename Index, typename PacketType> + PacketType packetOp(Index) const { + const int packetSize = internal::unpacket_traits<PacketType>::size; EIGEN_ALIGN_MAX T values[packetSize]; for (int i = 0; i < packetSize; ++i) { - values[i] = m_distribution(m_generator); + values[i] = m_distribution(*m_generator); } - return internal::pload<typename internal::packet_traits<T>::type>(values); + return internal::pload<PacketType>(values); } private: + // No assignment + NormalRandomGenerator& operator = (const NormalRandomGenerator&); + bool m_deterministic; mutable std::normal_distribution<T> m_distribution; - mutable std::mt19937 m_generator; + std::mt19937* m_generator; }; #elif defined (EIGEN_USE_GPU) && defined(__CUDACC__) && defined(__CUDA_ARCH__) @@ -605,11 +662,12 @@ template <> class NormalRandomGenerator<float> { curand_init(seed, tid, 0, &m_state); } template<typename Index> - __device__ float operator()(Index, Index = 0) const { + __device__ float operator()(Index) const { return curand_normal(&m_state); } - template<typename Index> - __device__ float4 packetOp(Index, Index = 0) const { + template<typename Index, typename PacketType> + __device__ float4 packetOp(Index) const { + EIGEN_STATIC_ASSERT((is_same<PacketType, float4>::value), YOU_MADE_A_PROGRAMMING_MISTAKE); return curand_normal4(&m_state); } @@ -634,11 +692,12 @@ template <> class NormalRandomGenerator<double> { curand_init(seed, tid, 0, &m_state); } template<typename Index> - __device__ double operator()(Index, Index = 0) const { + __device__ double operator()(Index) const { return curand_normal_double(&m_state); } - template<typename Index> - __device__ double2 packetOp(Index, Index = 0) const { + template<typename Index, typename PacketType> + __device__ double2 packetOp(Index) const { + EIGEN_STATIC_ASSERT((is_same<PacketType, double2>::value), YOU_MADE_A_PROGRAMMING_MISTAKE); return curand_normal2_double(&m_state); } @@ -663,7 +722,7 @@ template <> class NormalRandomGenerator<std::complex<float> > { curand_init(seed, tid, 0, &m_state); } template<typename Index> - __device__ std::complex<float> operator()(Index, Index = 0) const { + __device__ std::complex<float> operator()(Index) const { float4 vals = curand_normal4(&m_state); return std::complex<float>(vals.x, vals.y); } @@ -689,7 +748,7 @@ template <> class NormalRandomGenerator<std::complex<double> > { curand_init(seed, tid, 0, &m_state); } template<typename Index> - __device__ std::complex<double> operator()(Index, Index = 0) const { + __device__ std::complex<double> operator()(Index) const { double2 vals = curand_normal2_double(&m_state); return std::complex<double>(vals.x, vals.y); } @@ -703,6 +762,7 @@ template <> class NormalRandomGenerator<std::complex<double> > { template <typename T> class NormalRandomGenerator { public: + static const bool PacketAccess = false; NormalRandomGenerator(bool deterministic = true) : m_deterministic(deterministic) {} private: @@ -711,6 +771,13 @@ template <typename T> class NormalRandomGenerator { #endif +template <typename Scalar> +struct functor_traits<NormalRandomGenerator<Scalar> > { + enum { + PacketAccess = NormalRandomGenerator<Scalar>::PacketAccess + }; +}; + template <typename T, typename Index, size_t NumDims> class GaussianGenerator { diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorGenerator.h b/unsupported/Eigen/CXX11/src/Tensor/TensorGenerator.h index 9316c9831..e4154bd0b 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorGenerator.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorGenerator.h @@ -25,7 +25,6 @@ struct traits<TensorGeneratorOp<Generator, XprType> > : public traits<XprType> { typedef typename XprType::Scalar Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprTraits::StorageKind StorageKind; typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; @@ -55,10 +54,8 @@ class TensorGeneratorOp : public TensorBase<TensorGeneratorOp<Generator, XprType { public: typedef typename Eigen::internal::traits<TensorGeneratorOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorGeneratorOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; typedef typename Eigen::internal::nested<TensorGeneratorOp>::type Nested; typedef typename Eigen::internal::traits<TensorGeneratorOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorGeneratorOp>::Index Index; @@ -88,13 +85,15 @@ struct TensorEvaluator<const TensorGeneratorOp<Generator, ArgType>, Device> typedef typename TensorEvaluator<ArgType, Device>::Dimensions Dimensions; static const int NumDims = internal::array_size<Dimensions>::value; typedef typename XprType::Scalar Scalar; - + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; enum { IsAligned = false, - PacketAccess = (internal::packet_traits<Scalar>::size > 1), + PacketAccess = (internal::unpacket_traits<PacketReturnType>::size > 1), BlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -116,9 +115,6 @@ struct TensorEvaluator<const TensorGeneratorOp<Generator, ArgType>, Device> } } - typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* /*data*/) { diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorImagePatch.h b/unsupported/Eigen/CXX11/src/Tensor/TensorImagePatch.h index 11e510414..72594a05c 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorImagePatch.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorImagePatch.h @@ -32,7 +32,6 @@ struct traits<TensorImagePatchOp<Rows, Cols, XprType> > : public traits<XprType> { typedef typename internal::remove_const<typename XprType::Scalar>::type Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprTraits::StorageKind StorageKind; typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; @@ -60,10 +59,8 @@ class TensorImagePatchOp : public TensorBase<TensorImagePatchOp<Rows, Cols, XprT { public: typedef typename Eigen::internal::traits<TensorImagePatchOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorImagePatchOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; typedef typename Eigen::internal::nested<TensorImagePatchOp>::type Nested; typedef typename Eigen::internal::traits<TensorImagePatchOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorImagePatchOp>::Index Index; @@ -167,7 +164,8 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> IsAligned = false, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, - CoordAccess = NumDims == 5, + CoordAccess = false, + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -228,8 +226,8 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> m_outputRows = numext::ceil((m_input_rows_eff - m_patch_rows_eff + 1.f) / static_cast<float>(m_row_strides)); m_outputCols = numext::ceil((m_input_cols_eff - m_patch_cols_eff + 1.f) / static_cast<float>(m_col_strides)); // Calculate the padding - m_rowPaddingTop = ((m_outputRows - 1) * m_row_strides + m_patch_rows_eff - m_input_rows_eff) / 2; - m_colPaddingLeft = ((m_outputCols - 1) * m_col_strides + m_patch_cols_eff - m_input_cols_eff) / 2; + m_rowPaddingTop = numext::maxi<Index>(0, ((m_outputRows - 1) * m_row_strides + m_patch_rows_eff - m_input_rows_eff) / 2); + m_colPaddingLeft = numext::maxi<Index>(0, ((m_outputCols - 1) * m_col_strides + m_patch_cols_eff - m_input_cols_eff) / 2); break; case PADDING_SAME: m_outputRows = numext::ceil(m_input_rows_eff / static_cast<float>(m_row_strides)); @@ -296,8 +294,8 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> m_fastOtherStride = internal::TensorIntDivisor<Index>(m_otherStride); m_fastPatchStride = internal::TensorIntDivisor<Index>(m_patchStride); m_fastColStride = internal::TensorIntDivisor<Index>(m_colStride); - m_fastInputRowStride = internal::TensorIntDivisor<Index>(m_row_inflate_strides); - m_fastInputColStride = internal::TensorIntDivisor<Index>(m_col_inflate_strides); + m_fastInflateRowStride = internal::TensorIntDivisor<Index>(m_row_inflate_strides); + m_fastInflateColStride = internal::TensorIntDivisor<Index>(m_col_inflate_strides); m_fastInputColsEff = internal::TensorIntDivisor<Index>(m_input_cols_eff); // Number of patches in the width dimension. @@ -310,7 +308,7 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> } typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -338,7 +336,7 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> const Index colIndex = patch2DIndex / m_fastOutputRows; const Index colOffset = patchOffset / m_fastColStride; const Index inputCol = colIndex * m_col_strides + colOffset * m_in_col_strides - m_colPaddingLeft; - const Index origInputCol = (m_col_inflate_strides == 1) ? inputCol : ((inputCol >= 0) ? (inputCol / m_fastInputColStride) : 0); + const Index origInputCol = (m_col_inflate_strides == 1) ? inputCol : ((inputCol >= 0) ? (inputCol / m_fastInflateColStride) : 0); if (inputCol < 0 || inputCol >= m_input_cols_eff || ((m_col_inflate_strides != 1) && (inputCol != origInputCol * m_col_inflate_strides))) { return Scalar(m_paddingValue); @@ -348,7 +346,7 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> const Index rowIndex = patch2DIndex - colIndex * m_outputRows; const Index rowOffset = patchOffset - colOffset * m_colStride; const Index inputRow = rowIndex * m_row_strides + rowOffset * m_in_row_strides - m_rowPaddingTop; - const Index origInputRow = (m_row_inflate_strides == 1) ? inputRow : ((inputRow >= 0) ? (inputRow / m_fastInputRowStride) : 0); + const Index origInputRow = (m_row_inflate_strides == 1) ? inputRow : ((inputRow >= 0) ? (inputRow / m_fastInflateRowStride) : 0); if (inputRow < 0 || inputRow >= m_input_rows_eff || ((m_row_inflate_strides != 1) && (inputRow != origInputRow * m_row_inflate_strides))) { return Scalar(m_paddingValue); @@ -436,59 +434,6 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> Index rowInflateStride() const { return m_row_inflate_strides; } Index colInflateStride() const { return m_col_inflate_strides; } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(const array<Index, NumDims>& coords) const - { - // Location of the first element of the patch. - // ColMajor - // 0: d, 1: patch_rows, 2: patch_cols, 3: number of patches, 4: number of batches - // RowMajor - // 0: number of batches, 1: number of patches, 2: patch_cols , 3: patch_rows, 4: d - const Index patch2DIndex = coords[static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 3 : 1]; - - array<Index, NumDims-1> inputCoords; - Index input_col_idx = patch2DIndex / m_fastInputColsEff; - Index inputCol = input_col_idx + coords[1] * m_in_row_strides - m_rowPaddingTop; - Index inputRow = patch2DIndex - input_col_idx * m_input_cols_eff + coords[2] * m_in_col_strides - m_colPaddingLeft; - const Index origInputCol = (m_col_inflate_strides == 1) ? inputCol : ((inputCol >= 0) ? (inputCol / m_fastInputColStride) : 0); - const Index origInputRow = (m_row_inflate_strides == 1) ? inputRow : ((inputRow >= 0) ? (inputRow / m_fastInputRowStride) : 0); - if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { - inputCoords[0] = coords[0]; // depth - inputCoords[1] = origInputCol; - inputCoords[2] = origInputRow; - inputCoords[3] = coords[4]; // batch - } else { - inputCoords[3] = coords[4]; // depth - inputCoords[2] = origInputCol; - inputCoords[1] = origInputRow; - inputCoords[0] = coords[0]; // batch - } - // If the computed coordinates are outside the original image perimeter, return 0. - if (inputCol < 0 || inputCol >= m_input_cols_eff || inputRow < 0 || inputRow >= m_input_rows_eff || - ((m_col_inflate_strides != 1) && (inputCol != origInputCol * m_col_inflate_strides)) || - ((m_row_inflate_strides != 1) && (inputRow != origInputRow * m_row_inflate_strides))) { - return Scalar(m_paddingValue); - } - if (TensorEvaluator<ArgType, Device>::CoordAccess) { - return m_impl.coeff(inputCoords); - } else { - Index inputIndex; - if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { - inputIndex = - inputCoords[3] * m_patchInputStride + - inputCoords[2] * m_colInputStride + - inputCoords[1] * m_rowInputStride + - inputCoords[0]; - } else { - inputIndex = - inputCoords[1] * m_patchInputStride + - inputCoords[2] * m_colInputStride + - inputCoords[3] * m_rowInputStride + - inputCoords[4]; - } - return m_impl.coeff(inputIndex); - } - } - protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetWithPossibleZero(Index index) const { @@ -522,8 +467,8 @@ struct TensorEvaluator<const TensorImagePatchOp<Rows, Cols, ArgType>, Device> internal::TensorIntDivisor<Index> m_fastOtherStride; internal::TensorIntDivisor<Index> m_fastPatchStride; internal::TensorIntDivisor<Index> m_fastColStride; - internal::TensorIntDivisor<Index> m_fastInputRowStride; - internal::TensorIntDivisor<Index> m_fastInputColStride; + internal::TensorIntDivisor<Index> m_fastInflateRowStride; + internal::TensorIntDivisor<Index> m_fastInflateColStride; internal::TensorIntDivisor<Index> m_fastInputColsEff; Index m_rowInputStride; diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorIndexList.h b/unsupported/Eigen/CXX11/src/Tensor/TensorIndexList.h index 74ce6d0ec..985594bc8 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorIndexList.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorIndexList.h @@ -39,19 +39,36 @@ namespace Eigen { template <DenseIndex n> struct type2index { static const DenseIndex value = n; - constexpr operator DenseIndex() const { return n; } - void set(DenseIndex val) { + EIGEN_DEVICE_FUNC constexpr operator DenseIndex() const { return n; } + EIGEN_DEVICE_FUNC void set(DenseIndex val) { eigen_assert(val == n); } }; +template<DenseIndex n> struct NumTraits<type2index<n> > +{ + typedef DenseIndex Real; + enum { + IsComplex = 0, + RequireInitialization = false, + ReadCost = 1, + AddCost = 1, + MulCost = 1 + }; + + EIGEN_DEVICE_FUNC static inline Real epsilon() { return 0; } + EIGEN_DEVICE_FUNC static inline Real dummy_precision() { return 0; } + EIGEN_DEVICE_FUNC static inline Real highest() { return n; } + EIGEN_DEVICE_FUNC static inline Real lowest() { return n; } +}; + namespace internal { template <typename T> -void update_value(T& val, DenseIndex new_val) { +EIGEN_DEVICE_FUNC void update_value(T& val, DenseIndex new_val) { val = new_val; } template <DenseIndex n> -void update_value(type2index<n>& val, DenseIndex new_val) { +EIGEN_DEVICE_FUNC void update_value(type2index<n>& val, DenseIndex new_val) { val.set(new_val); } @@ -85,8 +102,8 @@ struct IndexTuple; template<typename T, typename... O> struct IndexTuple<T, O...> { - constexpr IndexTuple() : head(), others() { } - constexpr IndexTuple(const T& v, const O... o) : head(v), others(o...) { } + EIGEN_DEVICE_FUNC constexpr IndexTuple() : head(), others() { } + EIGEN_DEVICE_FUNC constexpr IndexTuple(const T& v, const O... o) : head(v), others(o...) { } constexpr static int count = 1 + sizeof...(O); T head; @@ -97,8 +114,8 @@ struct IndexTuple<T, O...> { template<typename T> struct IndexTuple<T> { - constexpr IndexTuple() : head() { } - constexpr IndexTuple(const T& v) : head(v) { } + EIGEN_DEVICE_FUNC constexpr IndexTuple() : head() { } + EIGEN_DEVICE_FUNC constexpr IndexTuple(const T& v) : head(v) { } constexpr static int count = 1; T head; @@ -114,33 +131,33 @@ struct IndexTupleExtractor<N, T, O...> { typedef typename IndexTupleExtractor<N-1, O...>::ValType ValType; - static constexpr ValType& get_val(IndexTuple<T, O...>& val) { + EIGEN_DEVICE_FUNC static constexpr ValType& get_val(IndexTuple<T, O...>& val) { return IndexTupleExtractor<N-1, O...>::get_val(val.others); } - static constexpr const ValType& get_val(const IndexTuple<T, O...>& val) { + EIGEN_DEVICE_FUNC static constexpr const ValType& get_val(const IndexTuple<T, O...>& val) { return IndexTupleExtractor<N-1, O...>::get_val(val.others); } template <typename V> - static void set_val(IndexTuple<T, O...>& val, V& new_val) { + EIGEN_DEVICE_FUNC static void set_val(IndexTuple<T, O...>& val, V& new_val) { IndexTupleExtractor<N-1, O...>::set_val(val.others, new_val); } }; - template<typename T, typename... O> - struct IndexTupleExtractor<0, T, O...> { +template<typename T, typename... O> + struct IndexTupleExtractor<0, T, O...> { - typedef T ValType; + typedef T ValType; - static constexpr ValType& get_val(IndexTuple<T, O...>& val) { + EIGEN_DEVICE_FUNC static constexpr ValType& get_val(IndexTuple<T, O...>& val) { return val.head; } - static constexpr const ValType& get_val(const IndexTuple<T, O...>& val) { + EIGEN_DEVICE_FUNC static constexpr const ValType& get_val(const IndexTuple<T, O...>& val) { return val.head; } template <typename V> - static void set_val(IndexTuple<T, O...>& val, V& new_val) { + EIGEN_DEVICE_FUNC static void set_val(IndexTuple<T, O...>& val, V& new_val) { val.head = new_val; } }; @@ -148,11 +165,11 @@ struct IndexTupleExtractor<N, T, O...> { template <int N, typename T, typename... O> -constexpr typename IndexTupleExtractor<N, T, O...>::ValType& array_get(IndexTuple<T, O...>& tuple) { +EIGEN_DEVICE_FUNC constexpr typename IndexTupleExtractor<N, T, O...>::ValType& array_get(IndexTuple<T, O...>& tuple) { return IndexTupleExtractor<N, T, O...>::get_val(tuple); } template <int N, typename T, typename... O> -constexpr const typename IndexTupleExtractor<N, T, O...>::ValType& array_get(const IndexTuple<T, O...>& tuple) { +EIGEN_DEVICE_FUNC constexpr const typename IndexTupleExtractor<N, T, O...>::ValType& array_get(const IndexTuple<T, O...>& tuple) { return IndexTupleExtractor<N, T, O...>::get_val(tuple); } template <typename T, typename... O> @@ -170,11 +187,11 @@ template <typename T, typename... O> template <DenseIndex Idx> struct tuple_coeff { template <typename... T> - static constexpr DenseIndex get(const DenseIndex i, const IndexTuple<T...>& t) { + EIGEN_DEVICE_FUNC static constexpr DenseIndex get(const DenseIndex i, const IndexTuple<T...>& t) { return array_get<Idx>(t) * (i == Idx) + tuple_coeff<Idx-1>::get(i, t) * (i != Idx); } template <typename... T> - static void set(const DenseIndex i, IndexTuple<T...>& t, const DenseIndex value) { + EIGEN_DEVICE_FUNC static void set(const DenseIndex i, IndexTuple<T...>& t, const DenseIndex value) { if (i == Idx) { update_value(array_get<Idx>(t), value); } else { @@ -183,19 +200,19 @@ struct tuple_coeff { } template <typename... T> - static constexpr bool value_known_statically(const DenseIndex i, const IndexTuple<T...>& t) { + EIGEN_DEVICE_FUNC static constexpr bool value_known_statically(const DenseIndex i, const IndexTuple<T...>& t) { return ((i == Idx) & is_compile_time_constant<typename IndexTupleExtractor<Idx, T...>::ValType>::value) || tuple_coeff<Idx-1>::value_known_statically(i, t); } template <typename... T> - static constexpr bool values_up_to_known_statically(const IndexTuple<T...>& t) { + EIGEN_DEVICE_FUNC static constexpr bool values_up_to_known_statically(const IndexTuple<T...>& t) { return is_compile_time_constant<typename IndexTupleExtractor<Idx, T...>::ValType>::value && tuple_coeff<Idx-1>::values_up_to_known_statically(t); } template <typename... T> - static constexpr bool values_up_to_statically_known_to_increase(const IndexTuple<T...>& t) { + EIGEN_DEVICE_FUNC static constexpr bool values_up_to_statically_known_to_increase(const IndexTuple<T...>& t) { return is_compile_time_constant<typename IndexTupleExtractor<Idx, T...>::ValType>::value && is_compile_time_constant<typename IndexTupleExtractor<Idx, T...>::ValType>::value && array_get<Idx>(t) > array_get<Idx-1>(t) && @@ -206,27 +223,27 @@ struct tuple_coeff { template <> struct tuple_coeff<0> { template <typename... T> - static constexpr DenseIndex get(const DenseIndex i, const IndexTuple<T...>& t) { + EIGEN_DEVICE_FUNC static constexpr DenseIndex get(const DenseIndex i, const IndexTuple<T...>& t) { // eigen_assert (i == 0); // gcc fails to compile assertions in constexpr return array_get<0>(t) * (i == 0); } template <typename... T> - static void set(const DenseIndex i, IndexTuple<T...>& t, const DenseIndex value) { + EIGEN_DEVICE_FUNC static void set(const DenseIndex i, IndexTuple<T...>& t, const DenseIndex value) { eigen_assert (i == 0); update_value(array_get<0>(t), value); } template <typename... T> - static constexpr bool value_known_statically(const DenseIndex i, const IndexTuple<T...>&) { + EIGEN_DEVICE_FUNC static constexpr bool value_known_statically(const DenseIndex i, const IndexTuple<T...>&) { return is_compile_time_constant<typename IndexTupleExtractor<0, T...>::ValType>::value & (i == 0); } template <typename... T> - static constexpr bool values_up_to_known_statically(const IndexTuple<T...>&) { + EIGEN_DEVICE_FUNC static constexpr bool values_up_to_known_statically(const IndexTuple<T...>&) { return is_compile_time_constant<typename IndexTupleExtractor<0, T...>::ValType>::value; } template <typename... T> - static constexpr bool values_up_to_statically_known_to_increase(const IndexTuple<T...>&) { + EIGEN_DEVICE_FUNC static constexpr bool values_up_to_statically_known_to_increase(const IndexTuple<T...>&) { return true; } }; @@ -235,7 +252,7 @@ struct tuple_coeff<0> { template<typename FirstType, typename... OtherTypes> - struct IndexList : internal::IndexTuple<FirstType, OtherTypes...> { +struct IndexList : internal::IndexTuple<FirstType, OtherTypes...> { EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC constexpr DenseIndex operator[] (const DenseIndex i) const { return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1>::get(i, *this); } @@ -246,18 +263,18 @@ template<typename FirstType, typename... OtherTypes> return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1>::set(i, *this, value); } - constexpr IndexList(const internal::IndexTuple<FirstType, OtherTypes...>& other) : internal::IndexTuple<FirstType, OtherTypes...>(other) { } - constexpr IndexList(FirstType& first, OtherTypes... other) : internal::IndexTuple<FirstType, OtherTypes...>(first, other...) { } - constexpr IndexList() : internal::IndexTuple<FirstType, OtherTypes...>() { } + EIGEN_DEVICE_FUNC constexpr IndexList(const internal::IndexTuple<FirstType, OtherTypes...>& other) : internal::IndexTuple<FirstType, OtherTypes...>(other) { } + EIGEN_DEVICE_FUNC constexpr IndexList(FirstType& first, OtherTypes... other) : internal::IndexTuple<FirstType, OtherTypes...>(first, other...) { } + EIGEN_DEVICE_FUNC constexpr IndexList() : internal::IndexTuple<FirstType, OtherTypes...>() { } - constexpr bool value_known_statically(const DenseIndex i) const { + EIGEN_DEVICE_FUNC constexpr bool value_known_statically(const DenseIndex i) const { return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1>::value_known_statically(i, *this); } - constexpr bool all_values_known_statically() const { + EIGEN_DEVICE_FUNC constexpr bool all_values_known_statically() const { return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1>::values_up_to_known_statically(*this); } - constexpr bool values_statically_known_to_increase() const { + EIGEN_DEVICE_FUNC constexpr bool values_statically_known_to_increase() const { return internal::tuple_coeff<internal::array_size<internal::IndexTuple<FirstType, OtherTypes...> >::value-1>::values_up_to_statically_known_to_increase(*this); } }; @@ -286,30 +303,30 @@ template<typename FirstType, typename... OtherTypes> struct array_size<const Ind static const size_t value = array_size<IndexTuple<FirstType, OtherTypes...> >::value; }; -template<DenseIndex N, typename FirstType, typename... OtherTypes> constexpr DenseIndex array_get(IndexList<FirstType, OtherTypes...>& a) { +template<DenseIndex N, typename FirstType, typename... OtherTypes> EIGEN_DEVICE_FUNC constexpr DenseIndex array_get(IndexList<FirstType, OtherTypes...>& a) { return IndexTupleExtractor<N, FirstType, OtherTypes...>::get_val(a); } -template<DenseIndex N, typename FirstType, typename... OtherTypes> constexpr DenseIndex array_get(const IndexList<FirstType, OtherTypes...>& a) { +template<DenseIndex N, typename FirstType, typename... OtherTypes> EIGEN_DEVICE_FUNC constexpr DenseIndex array_get(const IndexList<FirstType, OtherTypes...>& a) { return IndexTupleExtractor<N, FirstType, OtherTypes...>::get_val(a); } template <typename T> struct index_known_statically_impl { - static constexpr bool run(const DenseIndex) { + EIGEN_DEVICE_FUNC static constexpr bool run(const DenseIndex) { return false; } }; template <typename FirstType, typename... OtherTypes> struct index_known_statically_impl<IndexList<FirstType, OtherTypes...> > { - static constexpr bool run(const DenseIndex i) { + EIGEN_DEVICE_FUNC static constexpr bool run(const DenseIndex i) { return IndexList<FirstType, OtherTypes...>().value_known_statically(i); } }; template <typename FirstType, typename... OtherTypes> struct index_known_statically_impl<const IndexList<FirstType, OtherTypes...> > { - static constexpr bool run(const DenseIndex i) { + EIGEN_DEVICE_FUNC static constexpr bool run(const DenseIndex i) { return IndexList<FirstType, OtherTypes...>().value_known_statically(i); } }; @@ -324,14 +341,14 @@ struct all_indices_known_statically_impl { template <typename FirstType, typename... OtherTypes> struct all_indices_known_statically_impl<IndexList<FirstType, OtherTypes...> > { - static constexpr bool run() { + EIGEN_DEVICE_FUNC static constexpr bool run() { return IndexList<FirstType, OtherTypes...>().all_values_known_statically(); } }; template <typename FirstType, typename... OtherTypes> struct all_indices_known_statically_impl<const IndexList<FirstType, OtherTypes...> > { - static constexpr bool run() { + EIGEN_DEVICE_FUNC static constexpr bool run() { return IndexList<FirstType, OtherTypes...>().all_values_known_statically(); } }; @@ -339,21 +356,21 @@ struct all_indices_known_statically_impl<const IndexList<FirstType, OtherTypes.. template <typename T> struct indices_statically_known_to_increase_impl { - static constexpr bool run() { + EIGEN_DEVICE_FUNC static constexpr bool run() { return false; } }; template <typename FirstType, typename... OtherTypes> struct indices_statically_known_to_increase_impl<IndexList<FirstType, OtherTypes...> > { - static constexpr bool run() { + EIGEN_DEVICE_FUNC static constexpr bool run() { return Eigen::IndexList<FirstType, OtherTypes...>().values_statically_known_to_increase(); } }; template <typename FirstType, typename... OtherTypes> struct indices_statically_known_to_increase_impl<const IndexList<FirstType, OtherTypes...> > { - static constexpr bool run() { + EIGEN_DEVICE_FUNC static constexpr bool run() { return Eigen::IndexList<FirstType, OtherTypes...>().values_statically_known_to_increase(); } }; diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorInflation.h b/unsupported/Eigen/CXX11/src/Tensor/TensorInflation.h index ae9e9f751..368e6f685 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorInflation.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorInflation.h @@ -25,7 +25,6 @@ struct traits<TensorInflationOp<Strides, XprType> > : public traits<XprType> { typedef typename XprType::Scalar Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprTraits::StorageKind StorageKind; typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; @@ -53,10 +52,8 @@ class TensorInflationOp : public TensorBase<TensorInflationOp<Strides, XprType>, { public: typedef typename Eigen::internal::traits<TensorInflationOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorInflationOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; typedef typename Eigen::internal::nested<TensorInflationOp>::type Nested; typedef typename Eigen::internal::traits<TensorInflationOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorInflationOp>::Index Index; @@ -91,6 +88,7 @@ struct TensorEvaluator<const TensorInflationOp<Strides, ArgType>, Device> BlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -127,7 +125,8 @@ struct TensorEvaluator<const TensorInflationOp<Strides, ArgType>, Device> typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorInitializer.h b/unsupported/Eigen/CXX11/src/Tensor/TensorInitializer.h index ad2a1e6ac..2d223140e 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorInitializer.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorInitializer.h @@ -60,7 +60,7 @@ struct Initializer<Derived, 0> { typedef typename traits<Derived>::Scalar InitList; static void run(TensorEvaluator<Derived, DefaultDevice>& tensor, - Eigen::array<typename traits<Derived>::Index, traits<Derived>::NumDimensions>*/* indices*/, + Eigen::array<typename traits<Derived>::Index, traits<Derived>::NumDimensions>*, const InitList& v) { tensor.coeffRef(0) = v; } diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorIntDiv.h b/unsupported/Eigen/CXX11/src/Tensor/TensorIntDiv.h index b58173e58..33c6c1b0f 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorIntDiv.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorIntDiv.h @@ -36,17 +36,17 @@ namespace { #ifdef __CUDA_ARCH__ return (sizeof(T) == 8) ? __clzll(val) : __clz(val); #elif EIGEN_COMP_MSVC - DWORD leading_zeros = 0; - if (sizeof(T) == 8) { - _BitScanReverse64(&leading_zero, val); + unsigned long index; + if (sizeof(T) == 8) { + _BitScanReverse64(&index, val); + } else { + _BitScanReverse(&index, val); } - else { - _BitScanReverse(&leading_zero, val); - } - return leading_zeros; + return (sizeof(T) == 8) ? 63 - index : 31 - index; #else + EIGEN_STATIC_ASSERT(sizeof(unsigned long long) == 8, YOU_MADE_A_PROGRAMMING_MISTAKE); return (sizeof(T) == 8) ? - __builtin_clzl(static_cast<uint64_t>(val)) : + __builtin_clzll(static_cast<uint64_t>(val)) : __builtin_clz(static_cast<uint32_t>(val)); #endif } diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorLayoutSwap.h b/unsupported/Eigen/CXX11/src/Tensor/TensorLayoutSwap.h index f612bbd45..9b85914ff 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorLayoutSwap.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorLayoutSwap.h @@ -40,7 +40,6 @@ struct traits<TensorLayoutSwapOp<XprType> > : public traits<XprType> { typedef typename XprType::Scalar Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprTraits::StorageKind StorageKind; typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; @@ -70,10 +69,8 @@ class TensorLayoutSwapOp : public TensorBase<TensorLayoutSwapOp<XprType>, WriteA { public: typedef typename Eigen::internal::traits<TensorLayoutSwapOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorLayoutSwapOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename internal::remove_const<typename XprType::CoeffReturnType>::type CoeffReturnType; - typedef typename internal::remove_const<typename XprType::PacketReturnType>::type PacketReturnType; typedef typename Eigen::internal::nested<TensorLayoutSwapOp>::type Nested; typedef typename Eigen::internal::traits<TensorLayoutSwapOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorLayoutSwapOp>::Index Index; @@ -123,6 +120,7 @@ struct TensorEvaluator<const TensorLayoutSwapOp<ArgType>, Device> PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, Layout = (static_cast<int>(TensorEvaluator<ArgType, Device>::Layout) == static_cast<int>(ColMajor)) ? RowMajor : ColMajor, CoordAccess = false, // to be implemented + RawAccess = TensorEvaluator<ArgType, Device>::RawAccess }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -135,7 +133,7 @@ struct TensorEvaluator<const TensorLayoutSwapOp<ArgType>, Device> typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -189,7 +187,7 @@ template<typename ArgType, typename Device> typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(Index index) { diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorMap.h b/unsupported/Eigen/CXX11/src/Tensor/TensorMap.h index 4347bc2ff..9ebd9172b 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorMap.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorMap.h @@ -28,7 +28,6 @@ template<typename PlainObjectType, int Options_> class TensorMap : public Tensor typedef typename internal::traits<PlainObjectType>::StorageKind StorageKind; typedef typename internal::traits<PlainObjectType>::Index Index; typedef typename internal::traits<PlainObjectType>::Scalar Scalar; - typedef typename internal::packet_traits<Scalar>::type Packet; typedef typename NumTraits<Scalar>::Real RealScalar; typedef typename Base::CoeffReturnType CoeffReturnType; @@ -47,9 +46,9 @@ template<typename PlainObjectType, int Options_> class TensorMap : public Tensor enum { IsAligned = ((int(Options_)&Aligned)==Aligned), - PacketAccess = (internal::packet_traits<Scalar>::size > 1), Layout = PlainObjectType::Layout, CoordAccess = true, + RawAccess = true }; EIGEN_DEVICE_FUNC @@ -134,31 +133,32 @@ template<typename PlainObjectType, int Options_> class TensorMap : public Tensor return m_data[0]; } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE const Scalar& operator()(Index index) const + { + eigen_internal_assert(index >= 0 && index < size()); + return m_data[index]; + } + #ifdef EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const Scalar& operator()(Index firstIndex, IndexTypes... otherIndices) const + EIGEN_STRONG_INLINE const Scalar& operator()(Index firstIndex, Index secondIndex, IndexTypes... otherIndices) const { - EIGEN_STATIC_ASSERT(sizeof...(otherIndices) + 1 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) + EIGEN_STATIC_ASSERT(sizeof...(otherIndices) + 2 == NumIndices, YOU_MADE_A_PROGRAMMING_MISTAKE) if (PlainObjectType::Options&RowMajor) { - const Index index = m_dimensions.IndexOfRowMajor(array<Index, NumIndices>{{firstIndex, otherIndices...}}); + const Index index = m_dimensions.IndexOfRowMajor(array<Index, NumIndices>{{firstIndex, secondIndex, otherIndices...}}); return m_data[index]; } else { - const Index index = m_dimensions.IndexOfColMajor(array<Index, NumIndices>{{firstIndex, otherIndices...}}); + const Index index = m_dimensions.IndexOfColMajor(array<Index, NumIndices>{{firstIndex, secondIndex, otherIndices...}}); return m_data[index]; } } #else EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE const Scalar& operator()(Index index) const - { - eigen_internal_assert(index >= 0 && index < size()); - return m_data[index]; - } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& operator()(Index i0, Index i1) const { if (PlainObjectType::Options&RowMajor) { - const Index index = i1 + i0 * m_dimensions[0]; + const Index index = i1 + i0 * m_dimensions[1]; return m_data[index]; } else { const Index index = i0 + i1 * m_dimensions[0]; @@ -169,7 +169,7 @@ template<typename PlainObjectType, int Options_> class TensorMap : public Tensor EIGEN_STRONG_INLINE const Scalar& operator()(Index i0, Index i1, Index i2) const { if (PlainObjectType::Options&RowMajor) { - const Index index = i2 + m_dimensions[1] * (i1 + m_dimensions[0] * i0); + const Index index = i2 + m_dimensions[2] * (i1 + m_dimensions[1] * i0); return m_data[index]; } else { const Index index = i0 + m_dimensions[0] * (i1 + m_dimensions[1] * i2); @@ -220,32 +220,33 @@ template<typename PlainObjectType, int Options_> class TensorMap : public Tensor return m_data[0]; } + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Scalar& operator()(Index index) + { + eigen_internal_assert(index >= 0 && index < size()); + return m_data[index]; + } + #ifdef EIGEN_HAS_VARIADIC_TEMPLATES template<typename... IndexTypes> EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Scalar& operator()(Index firstIndex, IndexTypes... otherIndices) + EIGEN_STRONG_INLINE Scalar& operator()(Index firstIndex, Index secondIndex, IndexTypes... otherIndices) { - static_assert(sizeof...(otherIndices) + 1 == NumIndices || NumIndices == Dynamic, "Number of indices used to access a tensor coefficient must be equal to the rank of the tensor."); - const std::size_t NumDims = sizeof...(otherIndices) + 1; + static_assert(sizeof...(otherIndices) + 2 == NumIndices || NumIndices == Dynamic, "Number of indices used to access a tensor coefficient must be equal to the rank of the tensor."); + const std::size_t NumDims = sizeof...(otherIndices) + 2; if (PlainObjectType::Options&RowMajor) { - const Index index = m_dimensions.IndexOfRowMajor(array<Index, NumDims>{{firstIndex, otherIndices...}}); + const Index index = m_dimensions.IndexOfRowMajor(array<Index, NumDims>{{firstIndex, secondIndex, otherIndices...}}); return m_data[index]; } else { - const Index index = m_dimensions.IndexOfColMajor(array<Index, NumDims>{{firstIndex, otherIndices...}}); + const Index index = m_dimensions.IndexOfColMajor(array<Index, NumDims>{{firstIndex, secondIndex, otherIndices...}}); return m_data[index]; } } #else EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE Scalar& operator()(Index index) - { - eigen_internal_assert(index >= 0 && index < size()); - return m_data[index]; - } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& operator()(Index i0, Index i1) { if (PlainObjectType::Options&RowMajor) { - const Index index = i1 + i0 * m_dimensions[0]; + const Index index = i1 + i0 * m_dimensions[1]; return m_data[index]; } else { const Index index = i0 + i1 * m_dimensions[0]; @@ -256,7 +257,7 @@ template<typename PlainObjectType, int Options_> class TensorMap : public Tensor EIGEN_STRONG_INLINE Scalar& operator()(Index i0, Index i1, Index i2) { if (PlainObjectType::Options&RowMajor) { - const Index index = i2 + m_dimensions[1] * (i1 + m_dimensions[0] * i0); + const Index index = i2 + m_dimensions[2] * (i1 + m_dimensions[1] * i0); return m_data[index]; } else { const Index index = i0 + m_dimensions[0] * (i1 + m_dimensions[1] * i2); diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorMeta.h b/unsupported/Eigen/CXX11/src/Tensor/TensorMeta.h index 785321666..6af2d45d4 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorMeta.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorMeta.h @@ -24,6 +24,11 @@ const T2& choose(Cond<false>, const T1&, const T2& second) { return second; } +template <typename T> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE +T divup(const T x, const T y) { + return (x + y - 1) / y; +} + template <size_t n> struct max_n_1 { static const size_t size = n; }; @@ -36,7 +41,7 @@ template <> struct max_n_1<0> { template <typename Scalar, typename Device> struct PacketType { typedef typename internal::packet_traits<Scalar>::type type; - static const int size = internal::unpacket_traits<type>::size; + enum { size = internal::unpacket_traits<type>::size }; }; // For CUDA packet types when using a GpuDevice @@ -101,13 +106,18 @@ bool operator!=(const Tuple<U, V>& x, const Tuple<U, V>& y) { #ifdef EIGEN_HAS_SFINAE -namespace internal{ +namespace internal { template<typename IndexType, Index... Is> EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array<Index, sizeof...(Is)> customIndices2Array(IndexType& idx, numeric_list<Index, Is...>) { return { idx[Is]... }; } + template<typename IndexType> + EIGEN_CONSTEXPR EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + array<Index, 0> customIndices2Array(IndexType&, numeric_list<Index>) { + return array<Index, 0>(); + } /** Make an array (for index/dimensions) out of a custom index */ template<typename Index, std::size_t NumIndices, typename IndexType> diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorMorphing.h b/unsupported/Eigen/CXX11/src/Tensor/TensorMorphing.h index bdc86e0fa..a9c222ea0 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorMorphing.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorMorphing.h @@ -25,7 +25,6 @@ struct traits<TensorReshapingOp<NewDimensions, XprType> > : public traits<XprTyp { typedef typename XprType::Scalar Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprTraits::StorageKind StorageKind; typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; @@ -55,10 +54,7 @@ class TensorReshapingOp : public TensorBase<TensorReshapingOp<NewDimensions, Xpr { public: typedef typename Eigen::internal::traits<TensorReshapingOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorReshapingOp>::Packet Packet; - typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename internal::remove_const<typename XprType::CoeffReturnType>::type CoeffReturnType; - typedef typename internal::remove_const<typename XprType::PacketReturnType>::type PacketReturnType; typedef typename Eigen::internal::nested<TensorReshapingOp>::type Nested; typedef typename Eigen::internal::traits<TensorReshapingOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorReshapingOp>::Index Index; @@ -110,6 +106,7 @@ struct TensorEvaluator<const TensorReshapingOp<NewDimensions, ArgType>, Device> PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = TensorEvaluator<ArgType, Device>::RawAccess }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -123,7 +120,7 @@ struct TensorEvaluator<const TensorReshapingOp<NewDimensions, ArgType>, Device> typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -145,7 +142,7 @@ struct TensorEvaluator<const TensorReshapingOp<NewDimensions, ArgType>, Device> return m_impl.template packet<LoadMode>(index); } - EIGEN_DEVICE_FUNC CoeffReturnType* data() const { return m_impl.data(); } + EIGEN_DEVICE_FUNC Scalar* data() const { return const_cast<Scalar*>(m_impl.data()); } const TensorEvaluator<ArgType, Device>& impl() const { return m_impl; } @@ -170,6 +167,7 @@ template<typename NewDimensions, typename ArgType, typename Device> PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = TensorEvaluator<ArgType, Device>::RawAccess }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -179,7 +177,7 @@ template<typename NewDimensions, typename ArgType, typename Device> typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(Index index) { @@ -206,7 +204,6 @@ struct traits<TensorSlicingOp<StartIndices, Sizes, XprType> > : public traits<Xp { typedef typename XprType::Scalar Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprTraits::StorageKind StorageKind; typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; @@ -236,10 +233,7 @@ class TensorSlicingOp : public TensorBase<TensorSlicingOp<StartIndices, Sizes, X { public: typedef typename Eigen::internal::traits<TensorSlicingOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorSlicingOp>::Packet Packet; - typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; typedef typename Eigen::internal::nested<TensorSlicingOp>::type Nested; typedef typename Eigen::internal::traits<TensorSlicingOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorSlicingOp>::Index Index; @@ -316,7 +310,8 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi IsAligned = /*TensorEvaluator<ArgType, Device>::IsAligned*/false, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, - CoordAccess = TensorEvaluator<ArgType, Device>::CoordAccess, + CoordAccess = false, + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -358,7 +353,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef Sizes Dimensions; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -443,7 +438,7 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi return rslt; } else { - typename internal::remove_const<CoeffReturnType>::type values[packetSize]; + EIGEN_ALIGN_MAX typename internal::remove_const<CoeffReturnType>::type values[packetSize]; values[0] = m_impl.coeff(inputIndices[0]); values[packetSize-1] = m_impl.coeff(inputIndices[1]); for (int i = 1; i < packetSize-1; ++i) { @@ -454,15 +449,6 @@ struct TensorEvaluator<const TensorSlicingOp<StartIndices, Sizes, ArgType>, Devi } } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(const array<Index, NumDims>& coords) - { - array<Index, NumDims> inputCoords; - for (int i = 0; i < NumDims; ++i) { - inputCoords = coords[i] + this->m_offsets[i]; - } - return m_impl.coeff(inputCoords); - } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar* data() const { Scalar* result = m_impl.data(); if (result) { @@ -544,7 +530,8 @@ struct TensorEvaluator<TensorSlicingOp<StartIndices, Sizes, ArgType>, Device> IsAligned = /*TensorEvaluator<ArgType, Device>::IsAligned*/false, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, - CoordAccess = TensorEvaluator<ArgType, Device>::CoordAccess, + CoordAccess = false, + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -554,7 +541,7 @@ struct TensorEvaluator<TensorSlicingOp<StartIndices, Sizes, ArgType>, Device> typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef Sizes Dimensions; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(Index index) @@ -604,15 +591,6 @@ struct TensorEvaluator<TensorSlicingOp<StartIndices, Sizes, ArgType>, Device> } } } - - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(const array<Index, NumDims>& coords) - { - array<Index, NumDims> inputCoords; - for (int i = 0; i < NumDims; ++i) { - inputCoords = coords[i] + this->m_offsets[i]; - } - return this->m_impl.coeffRef(inputCoords); - } }; diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorPadding.h b/unsupported/Eigen/CXX11/src/Tensor/TensorPadding.h index 91e32d200..a595a0175 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorPadding.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorPadding.h @@ -16,7 +16,7 @@ namespace Eigen { * \ingroup CXX11_Tensor_Module * * \brief Tensor padding class. - * At the moment only 0-padding is supported. + * At the moment only padding with a constant value is supported. * */ namespace internal { @@ -25,7 +25,6 @@ struct traits<TensorPaddingOp<PaddingDimensions, XprType> > : public traits<XprT { typedef typename XprType::Scalar Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprTraits::StorageKind StorageKind; typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; @@ -55,19 +54,19 @@ class TensorPaddingOp : public TensorBase<TensorPaddingOp<PaddingDimensions, Xpr { public: typedef typename Eigen::internal::traits<TensorPaddingOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorPaddingOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; typedef typename Eigen::internal::nested<TensorPaddingOp>::type Nested; typedef typename Eigen::internal::traits<TensorPaddingOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorPaddingOp>::Index Index; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorPaddingOp(const XprType& expr, const PaddingDimensions& padding_dims) - : m_xpr(expr), m_padding_dims(padding_dims) {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorPaddingOp(const XprType& expr, const PaddingDimensions& padding_dims, const Scalar padding_value) + : m_xpr(expr), m_padding_dims(padding_dims), m_padding_value(padding_value) {} EIGEN_DEVICE_FUNC const PaddingDimensions& padding() const { return m_padding_dims; } + EIGEN_DEVICE_FUNC + Scalar padding_value() const { return m_padding_value; } EIGEN_DEVICE_FUNC const typename internal::remove_all<typename XprType::Nested>::type& @@ -76,6 +75,7 @@ class TensorPaddingOp : public TensorBase<TensorPaddingOp<PaddingDimensions, Xpr protected: typename XprType::Nested m_xpr; const PaddingDimensions m_padding_dims; + const Scalar m_padding_value; }; @@ -93,10 +93,11 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = true, + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) - : m_impl(op.expression(), device), m_padding(op.padding()) + : m_impl(op.expression(), device), m_padding(op.padding()), m_paddingValue(op.padding_value()) { // The padding op doesn't change the rank of the tensor. Directly padding a scalar would lead // to a vector, which doesn't make sense. Instead one should reshape the scalar into a vector @@ -130,7 +131,7 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -150,27 +151,27 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device for (int i = NumDims - 1; i > 0; --i) { const Index idx = index / m_outputStrides[i]; if (idx < m_padding[i].first || idx >= m_dimensions[i] - m_padding[i].second) { - return Scalar(0); + return m_paddingValue; } inputIndex += (idx - m_padding[i].first) * m_inputStrides[i]; index -= idx * m_outputStrides[i]; } if (index < m_padding[0].first || index >= m_dimensions[0] - m_padding[0].second) { - return Scalar(0); + return m_paddingValue; } inputIndex += (index - m_padding[0].first); } else { for (int i = 0; i < NumDims - 1; ++i) { const Index idx = index / m_outputStrides[i+1]; if (idx < m_padding[i].first || idx >= m_dimensions[i] - m_padding[i].second) { - return Scalar(0); + return m_paddingValue; } inputIndex += (idx - m_padding[i].first) * m_inputStrides[i]; index -= idx * m_outputStrides[i+1]; } if (index < m_padding[NumDims-1].first || index >= m_dimensions[NumDims-1] - m_padding[NumDims-1].second) { - return Scalar(0); + return m_paddingValue; } inputIndex += (index - m_padding[NumDims-1].first); } @@ -193,14 +194,14 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device { const Index idx = coords[0]; if (idx < m_padding[0].first || idx >= m_dimensions[0] - m_padding[0].second) { - return Scalar(0); + return m_paddingValue; } inputIndex = idx - m_padding[0].first; } for (int i = 1; i < NumDims; ++i) { const Index idx = coords[i]; if (idx < m_padding[i].first || idx >= m_dimensions[i] - m_padding[i].second) { - return Scalar(0); + return m_paddingValue; } inputIndex += (idx - m_padding[i].first) * m_inputStrides[i]; } @@ -208,14 +209,14 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device { const Index idx = coords[NumDims-1]; if (idx < m_padding[NumDims-1].first || idx >= m_dimensions[NumDims-1] - m_padding[NumDims-1].second) { - return Scalar(0); + return m_paddingValue; } inputIndex = idx - m_padding[NumDims-1].first; } for (int i = NumDims - 2; i >= 0; --i) { const Index idx = coords[i]; if (idx < m_padding[i].first || idx >= m_dimensions[i] - m_padding[i].second) { - return Scalar(0); + return m_paddingValue; } inputIndex += (idx - m_padding[i].first) * m_inputStrides[i]; } @@ -244,11 +245,11 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device if (last < lastPaddedLeft) { // all the coefficient are in the padding zone. - return internal::pset1<PacketReturnType>(Scalar(0)); + return internal::pset1<PacketReturnType>(m_paddingValue); } else if (first >= firstPaddedRight && last < lastPaddedRight) { // all the coefficient are in the padding zone. - return internal::pset1<PacketReturnType>(Scalar(0)); + return internal::pset1<PacketReturnType>(m_paddingValue); } else if (first >= lastPaddedLeft && last < firstPaddedRight) { // all the coefficient are between the 2 padding zones. @@ -270,11 +271,11 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device if (last < lastPaddedLeft) { // all the coefficient are in the padding zone. - return internal::pset1<PacketReturnType>(Scalar(0)); + return internal::pset1<PacketReturnType>(m_paddingValue); } else if (first >= firstPaddedRight && last < lastPaddedRight) { // all the coefficient are in the padding zone. - return internal::pset1<PacketReturnType>(Scalar(0)); + return internal::pset1<PacketReturnType>(m_paddingValue); } else if (first >= lastPaddedLeft && last < firstPaddedRight) { // all the coefficient are between the 2 padding zones. @@ -303,11 +304,11 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device if (last < lastPaddedLeft) { // all the coefficient are in the padding zone. - return internal::pset1<PacketReturnType>(Scalar(0)); + return internal::pset1<PacketReturnType>(m_paddingValue); } else if (first >= firstPaddedRight && last < lastPaddedRight) { // all the coefficient are in the padding zone. - return internal::pset1<PacketReturnType>(Scalar(0)); + return internal::pset1<PacketReturnType>(m_paddingValue); } else if (first >= lastPaddedLeft && last < firstPaddedRight) { // all the coefficient are between the 2 padding zones. @@ -329,11 +330,11 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device if (last < lastPaddedLeft) { // all the coefficient are in the padding zone. - return internal::pset1<PacketReturnType>(Scalar(0)); + return internal::pset1<PacketReturnType>(m_paddingValue); } else if (first >= firstPaddedRight && last < lastPaddedRight) { // all the coefficient are in the padding zone. - return internal::pset1<PacketReturnType>(Scalar(0)); + return internal::pset1<PacketReturnType>(m_paddingValue); } else if (first >= lastPaddedLeft && last < firstPaddedRight) { // all the coefficient are between the 2 padding zones. @@ -360,6 +361,8 @@ struct TensorEvaluator<const TensorPaddingOp<PaddingDimensions, ArgType>, Device array<Index, NumDims> m_inputStrides; TensorEvaluator<ArgType, Device> m_impl; PaddingDimensions m_padding; + + Scalar m_paddingValue; }; diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorPatch.h b/unsupported/Eigen/CXX11/src/Tensor/TensorPatch.h index 8fb53f4f2..0bf460f4e 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorPatch.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorPatch.h @@ -25,7 +25,6 @@ struct traits<TensorPatchOp<PatchDim, XprType> > : public traits<XprType> { typedef typename XprType::Scalar Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprTraits::StorageKind StorageKind; typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; @@ -55,10 +54,8 @@ class TensorPatchOp : public TensorBase<TensorPatchOp<PatchDim, XprType>, ReadOn { public: typedef typename Eigen::internal::traits<TensorPatchOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorPatchOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; typedef typename Eigen::internal::nested<TensorPatchOp>::type Nested; typedef typename Eigen::internal::traits<TensorPatchOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorPatchOp>::Index Index; @@ -93,7 +90,8 @@ struct TensorEvaluator<const TensorPatchOp<PatchDim, ArgType>, Device> IsAligned = false, PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, - CoordAccess = true, + CoordAccess = false, + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -140,7 +138,7 @@ struct TensorEvaluator<const TensorPatchOp<PatchDim, ArgType>, Device> } typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -247,56 +245,6 @@ struct TensorEvaluator<const TensorPatchOp<PatchDim, ArgType>, Device> } } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(const array<Index, NumDims>& coords) const - { - Index patch_coord_idx = Layout == ColMajor ? NumDims - 1 : 0; - // Location of the first element of the patch. - const Index patchIndex = coords[patch_coord_idx]; - - if (TensorEvaluator<ArgType, Device>::CoordAccess) { - array<Index, NumDims-1> inputCoords; - if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { - for (int i = NumDims - 2; i > 0; --i) { - const Index patchIdx = patchIndex / m_patchStrides[i]; - patchIndex -= patchIdx * m_patchStrides[i]; - const Index offsetIdx = coords[i]; - inputCoords[i] = coords[i] + patchIdx; - } - } else { - for (int i = 0; i < NumDims - 2; ++i) { - const Index patchIdx = patchIndex / m_patchStrides[i]; - patchIndex -= patchIdx * m_patchStrides[i]; - const Index offsetIdx = coords[i+1]; - inputCoords[i] = coords[i+1] + patchIdx; - } - } - Index coords_idx = Layout == ColMajor ? 0 : NumDims - 1; - inputCoords[0] = (patchIndex + coords[coords_idx]); - return m_impl.coeff(inputCoords); - } - else { - Index inputIndex = 0; - if (Layout == ColMajor) { - for (int i = NumDims - 2; i > 0; --i) { - const Index patchIdx = patchIndex / m_patchStrides[i]; - patchIndex -= patchIdx * m_patchStrides[i]; - const Index offsetIdx = coords[i]; - inputIndex += (patchIdx + offsetIdx) * m_inputStrides[i]; - } - } else { - for (int i = 0; i < NumDims - 2; ++i) { - const Index patchIdx = patchIndex / m_patchStrides[i]; - patchIndex -= patchIdx * m_patchStrides[i]; - const Index offsetIdx = coords[i+1]; - inputIndex += (patchIdx + offsetIdx) * m_inputStrides[i]; - } - } - Index coords_idx = Layout == ColMajor ? 0 : NumDims - 1; - inputIndex += (patchIndex + coords[coords_idx]); - return m_impl.coeff(inputIndex); - } - } - EIGEN_DEVICE_FUNC Scalar* data() const { return NULL; } protected: diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorReduction.h b/unsupported/Eigen/CXX11/src/Tensor/TensorReduction.h index bd15295b8..00f870328 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorReduction.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorReduction.h @@ -24,11 +24,13 @@ template<typename Op, typename Dims, typename XprType> struct traits<TensorReductionOp<Op, Dims, XprType> > : traits<XprType> { - typedef typename traits<XprType>::Scalar Scalar; - typedef typename internal::packet_traits<Scalar>::type Packet; - typedef typename traits<XprType>::StorageKind StorageKind; - typedef typename traits<XprType>::Index Index; + typedef traits<XprType> XprTraits; + typedef typename XprTraits::Scalar Scalar; + typedef typename XprTraits::StorageKind StorageKind; + typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; + static const int NumDimensions = XprTraits::NumDimensions - array_size<Dims>::value; + static const int Layout = XprTraits::Layout; }; template<typename Op, typename Dims, typename XprType> @@ -219,127 +221,146 @@ struct FullReducer { #ifdef EIGEN_USE_THREADS // Multithreaded full reducers -template <typename Eval, typename Op, bool Vectorizable = (Eval::InputPacketAccess & Op::PacketAccess)> +template <typename Self, typename Op, + bool vectorizable = (Self::InputPacketAccess & Op::PacketAccess)> struct FullReducerShard { - static void run(const Eval& eval, typename Eval::Index firstIndex, typename Eval::Index numValuesToReduce, Op& reducer, FullReducerShard* shard) { - - shard->saccum = reducer.initialize(); - for (typename Eval::Index j = 0; j < numValuesToReduce; ++j) { - reducer.reduce(eval.m_impl.coeff(firstIndex + j), &shard->saccum); - } - } - - typename Eval::CoeffReturnType saccum; -}; - -template <typename Eval, typename Op> -struct FullReducerShard<Eval, Op, true> { - static void run(const Eval& eval, typename Eval::Index firstIndex, typename Eval::Index numValuesToReduce, Op& reducer, FullReducerShard* shard) { - - const int packetSize = internal::unpacket_traits<typename Eval::PacketReturnType>::size; - const typename Eval::Index VectorizedSize = (numValuesToReduce / packetSize) * packetSize; - - shard->paccum = reducer.template initializePacket<typename Eval::PacketReturnType>(); - for (typename Eval::Index j = 0; j < VectorizedSize; j += packetSize) { - reducer.reducePacket(eval.m_impl.template packet<Unaligned>(firstIndex + j), &shard->paccum); - } - shard->saccum = reducer.initialize(); - for (typename Eval::Index j = VectorizedSize; j < numValuesToReduce; ++j) { - reducer.reduce(eval.m_impl.coeff(firstIndex + j), &shard->saccum); - } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Self& self, typename Self::Index firstIndex, + typename Self::Index numValuesToReduce, Op& reducer, + typename Self::CoeffReturnType* output) { + *output = InnerMostDimReducer<Self, Op, vectorizable>::reduce( + self, firstIndex, numValuesToReduce, reducer); } - - typename Eval::PacketReturnType paccum; - typename Eval::CoeffReturnType saccum; }; - template <typename Self, typename Op> struct FullReducer<Self, Op, ThreadPoolDevice, false> { static const bool HasOptimizedImplementation = !Op::IsStateful; + static const int PacketSize = + unpacket_traits<typename Self::PacketReturnType>::size; // launch one reducer per thread and accumulate the result. - static void run(const Self& self, Op& reducer, const ThreadPoolDevice& device, typename Self::CoeffReturnType* output) { + static void run(const Self& self, Op& reducer, const ThreadPoolDevice& device, + typename Self::CoeffReturnType* output) { typedef typename Self::Index Index; const Index num_coeffs = array_prod(self.m_impl.dimensions()); - const Index blocksize = std::floor<Index>(static_cast<float>(num_coeffs)/device.numThreads()); - const Index numblocks = blocksize > 0 ? num_coeffs / blocksize : 0; - eigen_assert(num_coeffs >= numblocks * blocksize); - - std::vector<Notification*> results; - results.reserve(numblocks); - std::vector<FullReducerShard<Self, Op, false> > shards; - shards.resize(numblocks); - for (Index i = 0; i < numblocks; ++i) { - results.push_back(device.enqueue(&FullReducerShard<Self, Op, false>::run, self, i*blocksize, blocksize, reducer, &shards[i])); - } - - FullReducerShard<Self, Op, false> finalShard; - if (numblocks * blocksize < num_coeffs) { - FullReducerShard<Self, Op, false>::run(self, numblocks * blocksize, num_coeffs - numblocks * blocksize, reducer, &finalShard); + if (num_coeffs == 0) { + *output = reducer.finalize(reducer.initialize()); + return; + } + const std::size_t num_threads = device.numThreads(); + if (num_threads == 1) { + *output = InnerMostDimReducer<Self, Op, false>::reduce(self, 0, num_coeffs, reducer); + return; } else { - finalShard.saccum = reducer.initialize(); - } - - for (Index i = 0; i < numblocks; ++i) { - wait_until_ready(results[i]); - delete results[i]; - } + const Index blocksize = std::floor<Index>(static_cast<float>(num_coeffs) / num_threads); + const unsigned int numblocks = blocksize > 0 ? static_cast<unsigned int>(num_coeffs / blocksize) : 0; + eigen_assert(num_coeffs >= static_cast<Index>(numblocks) * blocksize); + + Barrier barrier(numblocks); + MaxSizeVector<typename Self::CoeffReturnType> shards(numblocks, reducer.initialize()); + for (unsigned int i = 0; i < numblocks; ++i) { + device.enqueue_with_barrier(&barrier, &FullReducerShard<Self, Op, false>::run, self, + i * blocksize, blocksize, reducer, &shards[i]); + } - for (Index i = 0; i < numblocks; ++i) { - reducer.reduce(shards[i].saccum, &finalShard.saccum); + typename Self::CoeffReturnType finalShard; + if (static_cast<Index>(numblocks) * blocksize < num_coeffs) { + finalShard = InnerMostDimReducer<Self, Op, false>::reduce( + self, numblocks * blocksize, num_coeffs - numblocks * blocksize, reducer); + } else { + finalShard = reducer.initialize(); + } + barrier.Wait(); + for (unsigned int i = 0; i < numblocks; ++i) { + reducer.reduce(shards[i], &finalShard); + } + *output = reducer.finalize(finalShard); } - *output = reducer.finalize(finalShard.saccum); } }; template <typename Self, typename Op> struct FullReducer<Self, Op, ThreadPoolDevice, true> { static const bool HasOptimizedImplementation = !Op::IsStateful; + static const int PacketSize = + unpacket_traits<typename Self::PacketReturnType>::size; // launch one reducer per thread and accumulate the result. - static void run(const Self& self, Op& reducer, const ThreadPoolDevice& device, typename Self::CoeffReturnType* output) { + static void run(const Self& self, Op& reducer, const ThreadPoolDevice& device, + typename Self::CoeffReturnType* output) { typedef typename Self::Index Index; const Index num_coeffs = array_prod(self.m_impl.dimensions()); - const Index blocksize = std::floor<Index>(static_cast<float>(num_coeffs)/device.numThreads()); - const Index numblocks = blocksize > 0 ? num_coeffs / blocksize : 0; - eigen_assert(num_coeffs >= numblocks * blocksize); - - std::vector<Notification*> results; - results.reserve(numblocks); - std::vector<FullReducerShard<Self, Op, true> > shards; - shards.resize(numblocks); - for (Index i = 0; i < numblocks; ++i) { - results.push_back(device.enqueue(&FullReducerShard<Self, Op, true>::run, self, i*blocksize, blocksize, reducer, &shards[i])); - } - - FullReducerShard<Self, Op, true> finalShard; - if (numblocks * blocksize < num_coeffs) { - FullReducerShard<Self, Op, true>::run(self, numblocks * blocksize, num_coeffs - numblocks * blocksize, reducer, &finalShard); + if (num_coeffs == 0) { + *output = reducer.finalize(reducer.initialize()); + return; + } + const std::size_t num_threads = device.numThreads(); + if (num_threads == 1) { + *output = InnerMostDimReducer<Self, Op, true>::reduce(self, 0, num_coeffs, reducer); + return; + } + const Index blocksize = std::floor<Index>(static_cast<float>(num_coeffs) / num_threads); + const unsigned int numblocks = blocksize > 0 ? static_cast<unsigned int>(num_coeffs / blocksize) : 0; + eigen_assert(num_coeffs >= static_cast<Index>(numblocks) * blocksize); + + Barrier barrier(numblocks); + MaxSizeVector<typename Self::CoeffReturnType> shards(numblocks, reducer.initialize()); + for (unsigned int i = 0; i < numblocks; ++i) { + device.enqueue_with_barrier(&barrier, &FullReducerShard<Self, Op, true>::run, + self, i * blocksize, blocksize, reducer, + &shards[i]); + } + typename Self::CoeffReturnType finalShard; + if (static_cast<Index>(numblocks) * blocksize < num_coeffs) { + finalShard = InnerMostDimReducer<Self, Op, true>::reduce( + self, numblocks * blocksize, num_coeffs - numblocks * blocksize, reducer); } else { - finalShard.paccum = reducer.template initializePacket<typename Self::PacketReturnType>(); - finalShard.saccum = reducer.initialize(); + finalShard = reducer.initialize(); } - for (Index i = 0; i < numblocks; ++i) { - wait_until_ready(results[i]); - delete results[i]; + barrier.Wait(); + for (unsigned int i = 0; i < numblocks; ++i) { + reducer.reduce(shards[i], &finalShard); } + *output = reducer.finalize(finalShard); + } +}; - for (Index i = 0; i < numblocks; ++i) { - reducer.reducePacket(shards[i].paccum, &finalShard.paccum); - reducer.reduce(shards[i].saccum, &finalShard.saccum); - } +#endif - *output = reducer.finalizeBoth(finalShard.saccum, finalShard.paccum); + +// Default inner reducer +template <typename Self, typename Op, typename Device> +struct InnerReducer { + static const bool HasOptimizedImplementation = false; + + EIGEN_DEVICE_FUNC static bool run(const Self&, Op&, const Device&, typename Self::CoeffReturnType*, typename Self::Index, typename Self::Index) { + eigen_assert(false && "Not implemented"); + return true; + } +}; + +// Default outer reducer +template <typename Self, typename Op, typename Device> +struct OuterReducer { + static const bool HasOptimizedImplementation = false; + + EIGEN_DEVICE_FUNC static bool run(const Self&, Op&, const Device&, typename Self::CoeffReturnType*, typename Self::Index, typename Self::Index) { + eigen_assert(false && "Not implemented"); + return true; } }; -#endif #if defined(EIGEN_USE_GPU) && defined(__CUDACC__) template <int B, int N, typename S, typename R, typename I> __global__ void FullReductionKernel(R, const S, I, typename S::CoeffReturnType*); + +template <int NPT, typename S, typename R, typename I> +__global__ void InnerReductionKernel(R, const S, I, I, typename S::CoeffReturnType*); + +template <int NPT, typename S, typename R, typename I> +__global__ void OuterReductionKernel(R, const S, I, I, typename S::CoeffReturnType*); #endif } // end namespace internal @@ -349,10 +370,8 @@ template <typename Op, typename Dims, typename XprType> class TensorReductionOp : public TensorBase<TensorReductionOp<Op, Dims, XprType>, ReadOnlyAccessors> { public: typedef typename Eigen::internal::traits<TensorReductionOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorReductionOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename internal::remove_const<typename XprType::CoeffReturnType>::type CoeffReturnType; - typedef typename internal::remove_const<typename XprType::PacketReturnType>::type PacketReturnType; typedef typename Eigen::internal::nested<TensorReductionOp>::type Nested; typedef typename Eigen::internal::traits<TensorReductionOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorReductionOp>::Index Index; @@ -398,6 +417,7 @@ struct TensorEvaluator<const TensorReductionOp<Op, Dims, ArgType>, Device> PacketAccess = Self::InputPacketAccess && Op::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; static const bool ReducingInnerMostDims = internal::are_inner_most_dims<Dims, NumInputDims, Layout>::value; @@ -411,19 +431,18 @@ struct TensorEvaluator<const TensorReductionOp<Op, Dims, ArgType>, Device> EIGEN_STATIC_ASSERT((!ReducingInnerMostDims | !PreservingInnerMostDims | (NumReducedDims == NumInputDims)), YOU_MADE_A_PROGRAMMING_MISTAKE); - // Bitmap indicating if an input dimension is reduced or not. - array<bool, NumInputDims> reduced; + // Build the bitmap indicating if an input dimension is reduced or not. for (int i = 0; i < NumInputDims; ++i) { - reduced[i] = false; + m_reduced[i] = false; } for (int i = 0; i < NumReducedDims; ++i) { eigen_assert(op.dims()[i] >= 0); eigen_assert(op.dims()[i] < NumInputDims); - reduced[op.dims()[i]] = true; + m_reduced[op.dims()[i]] = true; } const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions(); - internal::DimInitializer<Dimensions>::run(input_dims, reduced, &m_dimensions, &m_reducedDims); + internal::DimInitializer<Dimensions>::run(input_dims, m_reduced, &m_dimensions, &m_reducedDims); // Precompute output strides. if (NumOutputDims > 0) { @@ -433,13 +452,13 @@ struct TensorEvaluator<const TensorReductionOp<Op, Dims, ArgType>, Device> m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1]; } } else { - m_outputStrides[NumOutputDims - 1] = 1; + m_outputStrides.back() = 1; for (int i = NumOutputDims - 2; i >= 0; --i) { m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1]; } } } - + // Precompute input strides. if (NumInputDims > 0) { array<Index, NumInputDims> input_strides; @@ -449,16 +468,16 @@ struct TensorEvaluator<const TensorReductionOp<Op, Dims, ArgType>, Device> input_strides[i] = input_strides[i-1] * input_dims[i-1]; } } else { - input_strides[NumInputDims - 1] = 1; + input_strides.back() = 1; for (int i = NumInputDims - 2; i >= 0; --i) { input_strides[i] = input_strides[i + 1] * input_dims[i + 1]; } } - + int outputIndex = 0; int reduceIndex = 0; for (int i = 0; i < NumInputDims; ++i) { - if (reduced[i]) { + if (m_reduced[i]) { m_reducedStrides[reduceIndex] = input_strides[i]; ++reduceIndex; } else { @@ -473,19 +492,19 @@ struct TensorEvaluator<const TensorReductionOp<Op, Dims, ArgType>, Device> m_preservedStrides[0] = internal::array_prod(input_dims); } } - + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } typedef typename internal::remove_const<typename XprType::CoeffReturnType>::type CoeffReturnType; - typedef typename internal::remove_const<typename XprType::PacketReturnType>::type PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; - EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(CoeffReturnType* data) { + EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool evalSubExprsIfNeeded(CoeffReturnType* data) { m_impl.evalSubExprsIfNeeded(NULL); // Use the FullReducer if possible. if (RunningFullReduction && internal::FullReducer<Self, Op, Device>::HasOptimizedImplementation && ((RunningOnGPU && (m_device.majorDeviceVersion() >= 3)) || - (internal::array_prod(m_impl.dimensions()) > 1024 * 1024))) { + (!RunningOnGPU && (internal::array_prod(m_impl.dimensions()) > 1024 * 1024)))) { bool need_assign = false; if (!data) { @@ -498,6 +517,41 @@ struct TensorEvaluator<const TensorReductionOp<Op, Dims, ArgType>, Device> internal::FullReducer<Self, Op, Device>::run(*this, reducer, m_device, data); return need_assign; } + + // Attempt to use an optimized reduction. + else if (RunningOnGPU && data && (m_device.majorDeviceVersion() >= 3)) { + bool reducing_inner_dims = true; + for (int i = 0; i < NumReducedDims; ++i) { + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + reducing_inner_dims &= m_reduced[i]; + } else { + reducing_inner_dims &= m_reduced[NumInputDims - 1 - i]; + } + } + if (internal::InnerReducer<Self, Op, Device>::HasOptimizedImplementation && + (reducing_inner_dims || ReducingInnerMostDims)) { + const Index num_values_to_reduce = internal::array_prod(m_reducedDims); + const Index num_coeffs_to_preserve = internal::array_prod(m_dimensions); + Op reducer(m_reducer); + return internal::InnerReducer<Self, Op, Device>::run(*this, reducer, m_device, data, num_values_to_reduce, num_coeffs_to_preserve); + } + + bool preserving_inner_dims = true; + for (int i = 0; i < NumReducedDims; ++i) { + if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { + preserving_inner_dims &= m_reduced[NumInputDims - 1 - i]; + } else { + preserving_inner_dims &= m_reduced[i]; + } + } + if (internal::OuterReducer<Self, Op, Device>::HasOptimizedImplementation && + preserving_inner_dims) { + const Index num_values_to_reduce = internal::array_prod(m_reducedDims); + const Index num_coeffs_to_preserve = internal::array_prod(m_dimensions); + Op reducer(m_reducer); + return internal::OuterReducer<Self, Op, Device>::run(*this, reducer, m_device, data, num_values_to_reduce, num_coeffs_to_preserve); + } + } return true; } @@ -579,6 +633,8 @@ struct TensorEvaluator<const TensorReductionOp<Op, Dims, ArgType>, Device> #endif #if defined(EIGEN_USE_GPU) && defined(__CUDACC__) template <int B, int N, typename S, typename R, typename I> friend void internal::FullReductionKernel(R, const S, I, typename S::CoeffReturnType*); + template <int NPT, typename S, typename R, typename I> friend void internal::InnerReductionKernel(R, const S, I, I, typename S::CoeffReturnType*); + template <int NPT, typename S, typename R, typename I> friend void internal::OuterReductionKernel(R, const S, I, I, typename S::CoeffReturnType*); #endif // Returns the Index in the input tensor of the first value that needs to be @@ -623,6 +679,8 @@ struct TensorEvaluator<const TensorReductionOp<Op, Dims, ArgType>, Device> return startInput; } + // Bitmap indicating if an input dimension is reduced or not. + array<bool, NumInputDims> m_reduced; // Dimensions of the output of the operation. Dimensions m_dimensions; // Precomputed strides for the output tensor. diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorReductionCuda.h b/unsupported/Eigen/CXX11/src/Tensor/TensorReductionCuda.h index 49102fca2..c33d54d6e 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorReductionCuda.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorReductionCuda.h @@ -7,8 +7,8 @@ // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -#ifndef EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_H -#define EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_H +#ifndef EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_CUDA_H +#define EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_CUDA_H namespace Eigen { namespace internal { @@ -76,26 +76,37 @@ __device__ inline void atomicReduce(T* output, T accum, SumReducer<T>&) { #endif } + +template <typename CoeffType, typename Index> +__global__ void ReductionInitKernel(const CoeffType val, Index num_preserved_coeffs, CoeffType* output) { + const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x; + const Index num_threads = blockDim.x * gridDim.x; + for (Index i = thread_id; i < num_preserved_coeffs; i += num_threads) { + output[i] = val; + } +} + template <int BlockSize, int NumPerThread, typename Self, typename Reducer, typename Index> __global__ void FullReductionKernel(Reducer reducer, const Self input, Index num_coeffs, typename Self::CoeffReturnType* output) { const Index first_index = blockIdx.x * BlockSize * NumPerThread + threadIdx.x; - if (first_index == 0) { + // Initialize the output value if it wasn't initialized by the ReductionInitKernel + if (gridDim.x == 1 && first_index == 0) { *output = reducer.initialize(); } typename Self::CoeffReturnType accum = reducer.initialize(); - for (Index i = 0; i < NumPerThread; ++i) { - const Index index = first_index + i * BlockSize; - if (index >= num_coeffs) { - break; - } + Index max_iter = numext::mini<Index>(num_coeffs - first_index, NumPerThread*BlockSize); + for (Index i = 0; i < max_iter; i+=BlockSize) { + const Index index = first_index + i; + eigen_assert(index < num_coeffs); typename Self::CoeffReturnType val = input.m_impl.coeff(index); reducer.reduce(val, &accum); } +#pragma unroll for (int offset = warpSize/2; offset > 0; offset /= 2) { reducer.reduce(__shfl_down(accum, offset), &accum); } @@ -115,26 +126,229 @@ struct FullReducer<Self, Op, GpuDevice, Vectorizable> { internal::is_same<typename Self::CoeffReturnType, float>::value; template <typename OutputType> - EIGEN_DEVICE_FUNC static void run(const Self& self, Op& reducer, const GpuDevice& device, OutputType* output) { + static EIGEN_DEVICE_FUNC void run(const Self&, Op&, const GpuDevice&, OutputType*) { assert(false && "Should only be called on floats"); } - EIGEN_DEVICE_FUNC static void run(const Self& self, Op& reducer, const GpuDevice& device, float* output) { + static EIGEN_DEVICE_FUNC void run(const Self& self, Op& reducer, const GpuDevice& device, float* output) { typedef typename Self::Index Index; const Index num_coeffs = array_prod(self.m_impl.dimensions()); const int block_size = 256; const int num_per_thread = 128; const int num_blocks = std::ceil(static_cast<float>(num_coeffs) / (block_size * num_per_thread)); - LAUNCH_CUDA_KERNEL((FullReductionKernel<block_size, num_per_thread>), + + if (num_blocks > 1) { + // We initialize the outputs outside the reduction kernel when we can't be sure that there + // won't be a race conditions between multiple thread blocks. + LAUNCH_CUDA_KERNEL((ReductionInitKernel<float, Index>), + 1, 32, 0, device, reducer.initialize(), 1, output); + } + + LAUNCH_CUDA_KERNEL((FullReductionKernel<block_size, num_per_thread, Self, Op, Index>), num_blocks, block_size, 0, device, reducer, self, num_coeffs, output); } }; + +template <int NumPerThread, typename Self, + typename Reducer, typename Index> +__global__ void InnerReductionKernel(Reducer reducer, const Self input, Index num_coeffs_to_reduce, Index num_preserved_coeffs, + typename Self::CoeffReturnType* output) { + eigen_assert(blockDim.y == 1); + eigen_assert(blockDim.z == 1); + eigen_assert(gridDim.y == 1); + eigen_assert(gridDim.z == 1); + + const int unroll_times = 16; + eigen_assert(NumPerThread % unroll_times == 0); + + const Index input_col_blocks = divup<Index>(num_coeffs_to_reduce, blockDim.x * NumPerThread); + const Index num_input_blocks = input_col_blocks * num_preserved_coeffs; + + const Index num_threads = blockDim.x * gridDim.x; + const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x; + + // Initialize the output values if they weren't initialized by the ReductionInitKernel + if (gridDim.x == 1) { + for (Index i = thread_id; i < num_preserved_coeffs; i += num_threads) { + output[i] = reducer.initialize(); + } + } + + for (Index i = blockIdx.x; i < num_input_blocks; i += gridDim.x) { + const Index row = i / input_col_blocks; + + if (row < num_preserved_coeffs) { + const Index col_block = i % input_col_blocks; + const Index col_begin = col_block * blockDim.x * NumPerThread + threadIdx.x; + + float reduced_val = reducer.initialize(); + + for (Index j = 0; j < NumPerThread; j += unroll_times) { + const Index last_col = col_begin + blockDim.x * (j + unroll_times - 1); + if (last_col >= num_coeffs_to_reduce) { + for (Index col = col_begin + blockDim.x * j; col < num_coeffs_to_reduce; col +=blockDim.x) { + const float val = input.m_impl.coeff(row * num_coeffs_to_reduce + col); + reducer.reduce(val, &reduced_val); + } + break; + } else { + // Faster version of the loop with no branches after unrolling. +#pragma unroll + for (int k = 0; k < unroll_times; ++k) { + const Index col = col_begin + blockDim.x * (j + k); + reducer.reduce(input.m_impl.coeff(row * num_coeffs_to_reduce + col), &reduced_val); + } + } + } + +#pragma unroll + for (int offset = warpSize/2; offset > 0; offset /= 2) { + reducer.reduce(__shfl_down(reduced_val, offset), &reduced_val); + } + + if ((threadIdx.x & (warpSize - 1)) == 0) { + atomicReduce(&(output[row]), reduced_val, reducer); + } + } + + __syncthreads(); + } +} + +template <typename Self, typename Op> +struct InnerReducer<Self, Op, GpuDevice> { + // Unfortunately nvidia doesn't support well exotic types such as complex, + // so reduce the scope of the optimized version of the code to the simple case + // of floats. + static const bool HasOptimizedImplementation = !Op::IsStateful && + internal::is_same<typename Self::CoeffReturnType, float>::value; + + template <typename Device, typename OutputType> + static EIGEN_DEVICE_FUNC bool run(const Self&, Op&, const Device&, OutputType*, typename Self::Index, typename Self::Index) { + assert(false && "Should only be called to reduce floats on a gpu device"); + return true; + } + + static EIGEN_DEVICE_FUNC bool run(const Self& self, Op& reducer, const GpuDevice& device, float* output, typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) { + typedef typename Self::Index Index; + + // It's faster to use the usual code. + if (num_coeffs_to_reduce <= 32) { + return true; + } + + const Index num_coeffs = num_coeffs_to_reduce * num_preserved_vals; + const int block_size = 256; + const int num_per_thread = 128; + const int dyn_blocks = divup<int>(num_coeffs, block_size * num_per_thread); + const int max_blocks = device.getNumCudaMultiProcessors() * + device.maxCudaThreadsPerMultiProcessor() / block_size; + const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks); + + if (num_blocks > 1) { + // We initialize the outputs outside the reduction kernel when we can't be sure that there + // won't be a race conditions between multiple thread blocks. + const int dyn_blocks = divup<int>(num_preserved_vals, 1024); + const int max_blocks = device.getNumCudaMultiProcessors() * + device.maxCudaThreadsPerMultiProcessor() / 1024; + const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks); + LAUNCH_CUDA_KERNEL((ReductionInitKernel<float, Index>), + num_blocks, 1024, 0, device, reducer.initialize(), + num_preserved_vals, output); + } + + LAUNCH_CUDA_KERNEL((InnerReductionKernel<num_per_thread, Self, Op, Index>), + num_blocks, block_size, 0, device, reducer, self, num_coeffs_to_reduce, num_preserved_vals, output); + + return false; + } +}; + + +template <int NumPerThread, typename Self, + typename Reducer, typename Index> +__global__ void OuterReductionKernel(Reducer reducer, const Self input, Index num_coeffs_to_reduce, Index num_preserved_coeffs, + typename Self::CoeffReturnType* output) { + const Index num_threads = blockDim.x * gridDim.x; + const Index thread_id = blockIdx.x * blockDim.x + threadIdx.x; + // Initialize the output values if they weren't initialized by the ReductionInitKernel + if (gridDim.x == 1) { + for (Index i = thread_id; i < num_preserved_coeffs; i += num_threads) { + output[i] = reducer.initialize(); + } + } + + // Do the reduction. + const Index max_iter = num_preserved_coeffs * divup<Index>(num_coeffs_to_reduce, NumPerThread); + for (Index i = thread_id; i < max_iter; i += num_threads) { + const Index input_col = i % num_preserved_coeffs; + const Index input_row = (i / num_preserved_coeffs) * NumPerThread; + typename Self::CoeffReturnType reduced_val = reducer.initialize(); + const Index max_row = numext::mini(input_row + NumPerThread, num_coeffs_to_reduce); + for (Index j = input_row; j < max_row; j++) { + typename Self::CoeffReturnType val = input.m_impl.coeff(j * num_preserved_coeffs + input_col); + reducer.reduce(val, &reduced_val); + } + atomicReduce(&(output[input_col]), reduced_val, reducer); + } +} + + +template <typename Self, typename Op> +struct OuterReducer<Self, Op, GpuDevice> { + // Unfortunately nvidia doesn't support well exotic types such as complex, + // so reduce the scope of the optimized version of the code to the simple case + // of floats. + static const bool HasOptimizedImplementation = !Op::IsStateful && + internal::is_same<typename Self::CoeffReturnType, float>::value; + + template <typename Device, typename OutputType> + static EIGEN_DEVICE_FUNC bool run(const Self&, Op&, const Device&, OutputType*, typename Self::Index, typename Self::Index) { + assert(false && "Should only be called to reduce floats on a gpu device"); + return true; + } + + static EIGEN_DEVICE_FUNC bool run(const Self& self, Op& reducer, const GpuDevice& device, float* output, typename Self::Index num_coeffs_to_reduce, typename Self::Index num_preserved_vals) { + typedef typename Self::Index Index; + + // It's faster to use the usual code. + if (num_coeffs_to_reduce <= 32) { + return true; + } + + const Index num_coeffs = num_coeffs_to_reduce * num_preserved_vals; + const int block_size = 256; + const int num_per_thread = 16; + const int dyn_blocks = divup<int>(num_coeffs, block_size * num_per_thread); + const int max_blocks = device.getNumCudaMultiProcessors() * + device.maxCudaThreadsPerMultiProcessor() / block_size; + const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks); + + if (num_blocks > 1) { + // We initialize the outputs in the reduction kernel itself when we don't have to worry + // about race conditions between multiple thread blocks. + const int dyn_blocks = divup<int>(num_preserved_vals, 1024); + const int max_blocks = device.getNumCudaMultiProcessors() * + device.maxCudaThreadsPerMultiProcessor() / 1024; + const int num_blocks = numext::mini<int>(max_blocks, dyn_blocks); + LAUNCH_CUDA_KERNEL((ReductionInitKernel<float, Index>), + num_blocks, 1024, 0, device, reducer.initialize(), + num_preserved_vals, output); + } + + LAUNCH_CUDA_KERNEL((OuterReductionKernel<num_per_thread, Self, Op, Index>), + num_blocks, block_size, 0, device, reducer, self, num_coeffs_to_reduce, num_preserved_vals, output); + + return false; + } +}; + #endif } // end namespace internal } // end namespace Eigen -#endif // EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_H +#endif // EIGEN_CXX11_TENSOR_TENSOR_REDUCTION_CUDA_H diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorRef.h b/unsupported/Eigen/CXX11/src/Tensor/TensorRef.h index 6b25b2ba0..bc92d9e6d 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorRef.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorRef.h @@ -125,7 +125,6 @@ template<typename PlainObjectType> class TensorRef : public TensorBase<TensorRef typedef typename internal::traits<PlainObjectType>::StorageKind StorageKind; typedef typename internal::traits<PlainObjectType>::Index Index; typedef typename internal::traits<PlainObjectType>::Scalar Scalar; - typedef typename internal::packet_traits<Scalar>::type Packet; typedef typename NumTraits<Scalar>::Real RealScalar; typedef typename Base::CoeffReturnType CoeffReturnType; typedef Scalar* PointerType; @@ -139,6 +138,7 @@ template<typename PlainObjectType> class TensorRef : public TensorBase<TensorRef PacketAccess = false, Layout = PlainObjectType::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_STRONG_INLINE TensorRef() : m_evaluator(NULL) { @@ -357,9 +357,8 @@ struct TensorEvaluator<const TensorRef<Derived>, Device> { typedef typename Derived::Index Index; typedef typename Derived::Scalar Scalar; - typedef typename Derived::Packet Packet; typedef typename Derived::Scalar CoeffReturnType; - typedef typename Derived::Packet PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef typename Derived::Dimensions Dimensions; enum { @@ -367,6 +366,7 @@ struct TensorEvaluator<const TensorRef<Derived>, Device> PacketAccess = false, Layout = TensorRef<Derived>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const TensorRef<Derived>& m, const Device&) @@ -402,9 +402,8 @@ struct TensorEvaluator<TensorRef<Derived>, Device> : public TensorEvaluator<cons { typedef typename Derived::Index Index; typedef typename Derived::Scalar Scalar; - typedef typename Derived::Packet Packet; typedef typename Derived::Scalar CoeffReturnType; - typedef typename Derived::Packet PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; typedef typename Derived::Dimensions Dimensions; typedef TensorEvaluator<const TensorRef<Derived>, Device> Base; @@ -412,6 +411,7 @@ struct TensorEvaluator<TensorRef<Derived>, Device> : public TensorEvaluator<cons enum { IsAligned = false, PacketAccess = false, + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(TensorRef<Derived>& m, const Device& d) : Base(m, d) diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorReverse.h b/unsupported/Eigen/CXX11/src/Tensor/TensorReverse.h index 10328c61f..96d92038c 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorReverse.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorReverse.h @@ -25,7 +25,6 @@ struct traits<TensorReverseOp<ReverseDimensions, { typedef typename XprType::Scalar Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprTraits::StorageKind StorageKind; typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; @@ -55,10 +54,8 @@ class TensorReverseOp : public TensorBase<TensorReverseOp<ReverseDimensions, { public: typedef typename Eigen::internal::traits<TensorReverseOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorReverseOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; typedef typename Eigen::internal::nested<TensorReverseOp>::type Nested; typedef typename Eigen::internal::traits<TensorReverseOp>::StorageKind StorageKind; @@ -113,6 +110,7 @@ struct TensorEvaluator<const TensorReverseOp<ReverseDimensions, ArgType>, Device PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, @@ -139,7 +137,7 @@ struct TensorEvaluator<const TensorReverseOp<ReverseDimensions, ArgType>, Device typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -239,6 +237,7 @@ struct TensorEvaluator<TensorReverseOp<ReverseDimensions, ArgType>, Device> PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -246,7 +245,7 @@ struct TensorEvaluator<TensorReverseOp<ReverseDimensions, ArgType>, Device> typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return this->m_dimensions; } diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorShuffling.h b/unsupported/Eigen/CXX11/src/Tensor/TensorShuffling.h index 15a22aa1b..c19833ea5 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorShuffling.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorShuffling.h @@ -25,7 +25,6 @@ struct traits<TensorShufflingOp<Shuffle, XprType> > : public traits<XprType> { typedef typename XprType::Scalar Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprTraits::StorageKind StorageKind; typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; @@ -55,10 +54,8 @@ class TensorShufflingOp : public TensorBase<TensorShufflingOp<Shuffle, XprType> { public: typedef typename Eigen::internal::traits<TensorShufflingOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorShufflingOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; typedef typename Eigen::internal::nested<TensorShufflingOp>::type Nested; typedef typename Eigen::internal::traits<TensorShufflingOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorShufflingOp>::Index Index; @@ -113,6 +110,7 @@ struct TensorEvaluator<const TensorShufflingOp<Shuffle, ArgType>, Device> PacketAccess = (internal::packet_traits<Scalar>::size > 1), Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -148,7 +146,7 @@ struct TensorEvaluator<const TensorShufflingOp<Shuffle, ArgType>, Device> } typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -225,6 +223,7 @@ struct TensorEvaluator<TensorShufflingOp<Shuffle, ArgType>, Device> enum { IsAligned = false, PacketAccess = (internal::packet_traits<Scalar>::size > 1), + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -232,7 +231,7 @@ struct TensorEvaluator<TensorShufflingOp<Shuffle, ArgType>, Device> { } typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(Index index) { diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorStorage.h b/unsupported/Eigen/CXX11/src/Tensor/TensorStorage.h index 98631fc7f..0e89033c4 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorStorage.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorStorage.h @@ -41,7 +41,10 @@ class TensorStorage<T, FixedDimensions, Options_> private: static const std::size_t Size = FixedDimensions::total_size; - EIGEN_ALIGN_MAX T m_data[Size]; + // Allocate an array of size at least one to prevent compiler warnings. + static const std::size_t MinSize = max_n_1<Size>::size; + EIGEN_ALIGN_MAX T m_data[MinSize]; + FixedDimensions m_dimensions; public: @@ -82,6 +85,13 @@ class TensorStorage<T, DSizes<IndexType, NumIndices_>, Options_> : m_data(internal::conditional_aligned_new_auto<T,(Options_&DontAlign)==0>(size)), m_dimensions(dimensions) { EIGEN_INTERNAL_TENSOR_STORAGE_CTOR_PLUGIN } +#ifdef EIGEN_HAS_VARIADIC_TEMPLATES + template <typename... DenseIndex> + EIGEN_DEVICE_FUNC TensorStorage(DenseIndex... indices) : m_dimensions(indices...) { + m_data = internal::conditional_aligned_new_auto<T,(Options_&DontAlign)==0>(internal::array_prod(m_dimensions)); + } +#endif + EIGEN_DEVICE_FUNC TensorStorage(const Self& other) : m_data(internal::conditional_aligned_new_auto<T,(Options_&DontAlign)==0>(internal::array_prod(other.m_dimensions))) , m_dimensions(other.m_dimensions) @@ -105,7 +115,6 @@ class TensorStorage<T, DSizes<IndexType, NumIndices_>, Options_> EIGEN_DEVICE_FUNC void resize(Index size, const array<Index, NumIndices_>& nbDimensions) { - eigen_assert(size >= 1); const Index currentSz = internal::array_prod(m_dimensions); if(size != currentSz) { diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorStriding.h b/unsupported/Eigen/CXX11/src/Tensor/TensorStriding.h index 97b6168a9..085f8fd3d 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorStriding.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorStriding.h @@ -25,7 +25,6 @@ struct traits<TensorStridingOp<Strides, XprType> > : public traits<XprType> { typedef typename XprType::Scalar Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprTraits::StorageKind StorageKind; typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; @@ -55,10 +54,8 @@ class TensorStridingOp : public TensorBase<TensorStridingOp<Strides, XprType> > { public: typedef typename Eigen::internal::traits<TensorStridingOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorStridingOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; typedef typename Eigen::internal::nested<TensorStridingOp>::type Nested; typedef typename Eigen::internal::traits<TensorStridingOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorStridingOp>::Index Index; @@ -112,6 +109,7 @@ struct TensorEvaluator<const TensorStridingOp<Strides, ArgType>, Device> PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -146,7 +144,7 @@ struct TensorEvaluator<const TensorStridingOp<Strides, ArgType>, Device> typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -258,6 +256,7 @@ struct TensorEvaluator<TensorStridingOp<Strides, ArgType>, Device> PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, Layout = TensorEvaluator<ArgType, Device>::Layout, CoordAccess = false, // to be implemented + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -265,7 +264,8 @@ struct TensorEvaluator<TensorStridingOp<Strides, ArgType>, Device> typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorTraits.h b/unsupported/Eigen/CXX11/src/Tensor/TensorTraits.h index 7a9568b36..2f06f8442 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorTraits.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorTraits.h @@ -20,7 +20,7 @@ class compute_tensor_flags enum { is_dynamic_size_storage = 1, - aligned_bit = + is_aligned = ( ((Options&DontAlign)==0) && ( #if EIGEN_MAX_STATIC_ALIGN_BYTES>0 @@ -35,12 +35,12 @@ class compute_tensor_flags 0 #endif ) - ) ? AlignedBit : 0, - packet_access_bit = packet_traits<Scalar>::Vectorizable && aligned_bit ? PacketAccessBit : 0 + ), + packet_access_bit = packet_traits<Scalar>::Vectorizable && is_aligned ? PacketAccessBit : 0 }; public: - enum { ret = packet_access_bit | aligned_bit}; + enum { ret = packet_access_bit}; }; @@ -86,7 +86,7 @@ struct traits<TensorMap<PlainObjectType, Options_> > static const int Layout = BaseTraits::Layout; enum { Options = Options_, - Flags = (BaseTraits::Flags & ~AlignedBit) | (Options&Aligned ? AlignedBit : 0), + Flags = BaseTraits::Flags, }; }; @@ -102,7 +102,7 @@ struct traits<TensorRef<PlainObjectType> > static const int Layout = BaseTraits::Layout; enum { Options = BaseTraits::Options, - Flags = (BaseTraits::Flags & ~AlignedBit) | (Options&Aligned ? AlignedBit : 0), + Flags = BaseTraits::Flags, }; }; diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorUInt128.h b/unsupported/Eigen/CXX11/src/Tensor/TensorUInt128.h index f5cca0ad7..3e56589c3 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorUInt128.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorUInt128.h @@ -33,18 +33,34 @@ struct TensorUInt128 HIGH high; LOW low; + template<typename OTHER_HIGH, typename OTHER_LOW> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE - TensorUInt128(int x) : high(0), low(x) { - eigen_assert(x >= 0); + TensorUInt128(const TensorUInt128<OTHER_HIGH, OTHER_LOW>& other) : high(other.high), low(other.low) { + EIGEN_STATIC_ASSERT(sizeof(OTHER_HIGH) <= sizeof(HIGH), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT(sizeof(OTHER_LOW) <= sizeof(LOW), YOU_MADE_A_PROGRAMMING_MISTAKE); } + + template<typename OTHER_HIGH, typename OTHER_LOW> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE - TensorUInt128(int64_t x) : high(0), low(x) { - eigen_assert(x >= 0); + TensorUInt128& operator = (const TensorUInt128<OTHER_HIGH, OTHER_LOW>& other) { + EIGEN_STATIC_ASSERT(sizeof(OTHER_HIGH) <= sizeof(HIGH), YOU_MADE_A_PROGRAMMING_MISTAKE); + EIGEN_STATIC_ASSERT(sizeof(OTHER_LOW) <= sizeof(LOW), YOU_MADE_A_PROGRAMMING_MISTAKE); + high = other.high; + low = other.low; + return *this; } + + template<typename T> EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE - TensorUInt128(uint64_t x) : high(0), low(x) { } + explicit TensorUInt128(const T& x) : high(0), low(x) { + typedef typename conditional<sizeof(T) == 8, uint64_t, uint32_t>::type UnsignedT; + typedef typename conditional<sizeof(LOW) == 8, uint64_t, uint32_t>::type UnsignedLow; + eigen_assert(static_cast<UnsignedT>(x) <= static_cast<UnsignedLow>(NumTraits<LOW>::highest())); + eigen_assert(x >= 0); + } + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE - TensorUInt128(uint64_t y, uint64_t x) : high(y), low(x) { } + TensorUInt128(HIGH y, LOW x) : high(y), low(x) { } EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE operator LOW() const { return low; diff --git a/unsupported/Eigen/CXX11/src/Tensor/TensorVolumePatch.h b/unsupported/Eigen/CXX11/src/Tensor/TensorVolumePatch.h index 6625c66d5..5bdfbad46 100644 --- a/unsupported/Eigen/CXX11/src/Tensor/TensorVolumePatch.h +++ b/unsupported/Eigen/CXX11/src/Tensor/TensorVolumePatch.h @@ -27,7 +27,6 @@ struct traits<TensorVolumePatchOp<Planes, Rows, Cols, XprType> > : public traits { typedef typename internal::remove_const<typename XprType::Scalar>::type Scalar; typedef traits<XprType> XprTraits; - typedef typename packet_traits<Scalar>::type Packet; typedef typename XprTraits::StorageKind StorageKind; typedef typename XprTraits::Index Index; typedef typename XprType::Nested Nested; @@ -55,10 +54,8 @@ class TensorVolumePatchOp : public TensorBase<TensorVolumePatchOp<Planes, Rows, { public: typedef typename Eigen::internal::traits<TensorVolumePatchOp>::Scalar Scalar; - typedef typename Eigen::internal::traits<TensorVolumePatchOp>::Packet Packet; typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; typedef typename Eigen::internal::nested<TensorVolumePatchOp>::type Nested; typedef typename Eigen::internal::traits<TensorVolumePatchOp>::StorageKind StorageKind; typedef typename Eigen::internal::traits<TensorVolumePatchOp>::Index Index; @@ -180,7 +177,8 @@ struct TensorEvaluator<const TensorVolumePatchOp<Planes, Rows, Cols, ArgType>, D PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess, BlockAccess = false, Layout = TensorEvaluator<ArgType, Device>::Layout, - CoordAccess = NumDims == 6, + CoordAccess = false, + RawAccess = false }; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) @@ -339,7 +337,7 @@ struct TensorEvaluator<const TensorVolumePatchOp<Planes, Rows, Cols, ArgType>, D } typedef typename XprType::CoeffReturnType CoeffReturnType; - typedef typename XprType::PacketReturnType PacketReturnType; + typedef typename PacketType<CoeffReturnType, Device>::type PacketReturnType; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; } @@ -517,79 +515,6 @@ struct TensorEvaluator<const TensorVolumePatchOp<Planes, Rows, Cols, ArgType>, D Index rowInflateStride() const { return m_row_inflate_strides; } Index colInflateStride() const { return m_col_inflate_strides; } - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(const array<Index, NumDims>& coords) const - { - // ColMajor - // 0: depth, 1: patch_planes, 2: patch_rows, 3: patch_cols, 4: number of patches, 5: batches - // RowMajor - // 0: batches, 1: number of patches, 2: patch_cols , 3: patch_rows, 4: patch_planes, 5: depth - const Index patch3DIndex = coords[static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 4 : 1]; - const Index colOffset = coords[static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 3 : 2]; - const Index rowOffset= coords[static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 2 : 3]; - const Index planeOffset = coords[static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 1 : 4]; - - array<Index, NumDims-1> inputCoords; - - const Index colIndex = patch3DIndex / m_fastOutputPlanesRows; - const Index inputCol = colIndex * m_col_strides + colOffset * m_in_col_strides - m_colPaddingLeft; - const Index origInputCol = (m_col_inflate_strides == 1) ? inputCol : ((inputCol >= 0) ? (inputCol / m_fastInputColStride) : 0); - if (inputCol < 0 || inputCol >= m_input_cols_eff || - ((m_col_inflate_strides != 1) && (inputCol != origInputCol * m_col_inflate_strides))) { - return Scalar(m_paddingValue); - } - - const Index rowIndex = (patch3DIndex - colIndex * m_outputPlanesRows) / m_fastOutputPlanes; - const Index inputRow = rowIndex * m_row_strides + rowOffset * m_in_row_strides - m_rowPaddingTop; - const Index origInputRow = (m_row_inflate_strides == 1) ? inputRow : ((inputRow >= 0) ? (inputRow / m_fastInputRowStride) : 0); - if (inputRow < 0 || inputRow >= m_input_rows_eff || - ((m_row_inflate_strides != 1) && (inputRow != origInputRow * m_row_inflate_strides))) { - return Scalar(m_paddingValue); - } - - const Index planeIndex = patch3DIndex - colIndex * m_outputPlanesRows - rowIndex * m_outputRows; - const Index inputPlane = planeIndex * m_plane_strides + planeOffset * m_in_plane_strides - m_planePaddingTop; - const Index origInputPlane = (m_plane_inflate_strides == 1) ? inputPlane : ((inputPlane >= 0) ? (inputPlane / m_fastInputPlaneStride) : 0); - if (inputPlane < 0 || inputPlane >= m_input_planes_eff || - ((m_plane_inflate_strides != 1) && (inputPlane != origInputPlane * m_plane_inflate_strides))) { - return Scalar(m_paddingValue); - } - - if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { - inputCoords[0] = coords[0]; // depth - inputCoords[1] = origInputPlane; - inputCoords[2] = origInputRow; - inputCoords[3] = origInputCol; - inputCoords[4] = coords[5]; // batch - } else { - inputCoords[4] = coords[5]; // depth - inputCoords[3] = origInputPlane; - inputCoords[2] = origInputRow; - inputCoords[1] = origInputCol; - inputCoords[0] = coords[0]; // batch - } - if (TensorEvaluator<ArgType, Device>::CoordAccess) { - return m_impl.coeff(inputCoords); - } else { - Index inputIndex; - if (static_cast<int>(Layout) == static_cast<int>(ColMajor)) { - inputIndex = - inputCoords[4] * m_otherInputStride + - inputCoords[3] * m_colInputStride + - inputCoords[2] * m_rowInputStride + - inputCoords[1] * m_planeInputStride + - inputCoords[0]; - } else { - inputIndex = - inputCoords[0] * m_otherInputStride + - inputCoords[1] * m_colInputStride + - inputCoords[2] * m_rowInputStride + - inputCoords[3] * m_planeInputStride + - inputCoords[4]; - } - return m_impl.coeff(inputIndex); - } - } - protected: EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetWithPossibleZero(Index index) const { |