diff options
122 files changed, 8441 insertions, 1718 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 96d6c8701..dac091b49 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -143,7 +143,7 @@ if(NOT MSVC) ei_add_cxx_compiler_flag("-Wpointer-arith") ei_add_cxx_compiler_flag("-Wwrite-strings") ei_add_cxx_compiler_flag("-Wformat-security") - ei_add_cxx_compiler_flag("-Wshorten-64-to-32") +# ei_add_cxx_compiler_flag("-Wshorten-64-to-32") ei_add_cxx_compiler_flag("-Wenum-conversion") ei_add_cxx_compiler_flag("-Wc++11-extensions") diff --git a/Eigen/Core b/Eigen/Core index 776b7faf3..239bc6990 100644 --- a/Eigen/Core +++ b/Eigen/Core @@ -11,6 +11,13 @@ #ifndef EIGEN_CORE_H #define EIGEN_CORE_H +// EIGEN_TEST_EVALUATORS => EIGEN_ENABLE_EVALUATORS +#ifdef EIGEN_TEST_EVALUATORS +#ifndef EIGEN_ENABLE_EVALUATORS +#define EIGEN_ENABLE_EVALUATORS +#endif +#endif + // first thing Eigen does: stop the compiler from committing suicide #include "src/Core/util/DisableStupidWarnings.h" @@ -266,8 +273,8 @@ using std::ptrdiff_t; */ #include "src/Core/util/Constants.h" -#include "src/Core/util/ForwardDeclarations.h" #include "src/Core/util/Meta.h" +#include "src/Core/util/ForwardDeclarations.h" #include "src/Core/util/StaticAssert.h" #include "src/Core/util/XprHelper.h" #include "src/Core/util/Memory.h" @@ -300,6 +307,7 @@ using std::ptrdiff_t; #include "src/Core/functors/UnaryFunctors.h" #include "src/Core/functors/NullaryFunctors.h" #include "src/Core/functors/StlFunctors.h" +#include "src/Core/functors/AssignmentFunctors.h" #include "src/Core/DenseCoeffsBase.h" #include "src/Core/DenseBase.h" @@ -307,11 +315,9 @@ using std::ptrdiff_t; #include "src/Core/EigenBase.h" #ifdef EIGEN_ENABLE_EVALUATORS -#include "src/Core/functors/AssignmentFunctors.h" #include "src/Core/Product.h" #include "src/Core/CoreEvaluators.h" #include "src/Core/AssignEvaluator.h" -#include "src/Core/ProductEvaluators.h" #endif #ifndef EIGEN_PARSED_BY_DOXYGEN // work around Doxygen bug triggered by Assign.h r814874 @@ -323,7 +329,10 @@ using std::ptrdiff_t; #include "src/Core/util/BlasUtil.h" #include "src/Core/DenseStorage.h" #include "src/Core/NestByValue.h" +#ifndef EIGEN_ENABLE_EVALUATORS #include "src/Core/ForceAlignedAccess.h" +#include "src/Core/Flagged.h" +#endif #include "src/Core/ReturnByValue.h" #include "src/Core/NoAlias.h" #include "src/Core/PlainObjectBase.h" @@ -336,12 +345,12 @@ using std::ptrdiff_t; #include "src/Core/SelfCwiseBinaryOp.h" #include "src/Core/Dot.h" #include "src/Core/StableNorm.h" -#include "src/Core/MapBase.h" #include "src/Core/Stride.h" +#include "src/Core/MapBase.h" #include "src/Core/Map.h" +#include "src/Core/Ref.h" #include "src/Core/Block.h" #include "src/Core/VectorBlock.h" -#include "src/Core/Ref.h" #include "src/Core/Transpose.h" #include "src/Core/DiagonalMatrix.h" #include "src/Core/Diagonal.h" @@ -354,14 +363,20 @@ using std::ptrdiff_t; #include "src/Core/IO.h" #include "src/Core/Swap.h" #include "src/Core/CommaInitializer.h" -#include "src/Core/Flagged.h" #include "src/Core/ProductBase.h" #include "src/Core/GeneralProduct.h" +#ifdef EIGEN_ENABLE_EVALUATORS +#include "src/Core/Solve.h" +#include "src/Core/Inverse.h" +#endif #include "src/Core/TriangularMatrix.h" #include "src/Core/SelfAdjointView.h" #include "src/Core/products/GeneralBlockPanelKernel.h" #include "src/Core/products/Parallelizer.h" #include "src/Core/products/CoeffBasedProduct.h" +#ifdef EIGEN_ENABLE_EVALUATORS +#include "src/Core/ProductEvaluators.h" +#endif #include "src/Core/products/GeneralMatrixVector.h" #include "src/Core/products/GeneralMatrixMatrix.h" #include "src/Core/SolveTriangular.h" @@ -25,7 +25,7 @@ #include "src/LU/PartialPivLU_MKL.h" #endif #include "src/LU/Determinant.h" -#include "src/LU/Inverse.h" +#include "src/LU/InverseImpl.h" // Use the SSE optimized version whenever possible. At the moment the // SSE version doesn't compile when AVX is enabled diff --git a/Eigen/SparseCore b/Eigen/SparseCore index 9b5be5e15..41cae58b0 100644 --- a/Eigen/SparseCore +++ b/Eigen/SparseCore @@ -35,28 +35,29 @@ struct Sparse {}; #include "src/SparseCore/SparseUtil.h" #include "src/SparseCore/SparseMatrixBase.h" +#include "src/SparseCore/SparseAssign.h" #include "src/SparseCore/CompressedStorage.h" #include "src/SparseCore/AmbiVector.h" #include "src/SparseCore/SparseMatrix.h" #include "src/SparseCore/MappedSparseMatrix.h" #include "src/SparseCore/SparseVector.h" -#include "src/SparseCore/SparseBlock.h" -#include "src/SparseCore/SparseTranspose.h" #include "src/SparseCore/SparseCwiseUnaryOp.h" #include "src/SparseCore/SparseCwiseBinaryOp.h" +#include "src/SparseCore/SparseTranspose.h" +#include "src/SparseCore/SparseBlock.h" #include "src/SparseCore/SparseDot.h" -#include "src/SparseCore/SparsePermutation.h" #include "src/SparseCore/SparseRedux.h" -#include "src/SparseCore/SparseFuzzy.h" +#include "src/SparseCore/SparseView.h" +#include "src/SparseCore/SparseDiagonalProduct.h" #include "src/SparseCore/ConservativeSparseSparseProduct.h" #include "src/SparseCore/SparseSparseProductWithPruning.h" #include "src/SparseCore/SparseProduct.h" #include "src/SparseCore/SparseDenseProduct.h" -#include "src/SparseCore/SparseDiagonalProduct.h" -#include "src/SparseCore/SparseTriangularView.h" #include "src/SparseCore/SparseSelfAdjointView.h" +#include "src/SparseCore/SparseTriangularView.h" #include "src/SparseCore/TriangularSolver.h" -#include "src/SparseCore/SparseView.h" +#include "src/SparseCore/SparsePermutation.h" +#include "src/SparseCore/SparseFuzzy.h" #include "src/Core/util/ReenableStupidWarnings.h" diff --git a/Eigen/src/Cholesky/LDLT.h b/Eigen/src/Cholesky/LDLT.h index aa9784e54..f621323a3 100644 --- a/Eigen/src/Cholesky/LDLT.h +++ b/Eigen/src/Cholesky/LDLT.h @@ -174,6 +174,17 @@ template<typename _MatrixType, int _UpLo> class LDLT * * \sa MatrixBase::ldlt(), SelfAdjointView::ldlt() */ +#ifdef EIGEN_TEST_EVALUATORS + template<typename Rhs> + inline const Solve<LDLT, Rhs> + solve(const MatrixBase<Rhs>& b) const + { + eigen_assert(m_isInitialized && "LDLT is not initialized."); + eigen_assert(m_matrix.rows()==b.rows() + && "LDLT::solve(): invalid number of rows of the right hand side matrix b"); + return Solve<LDLT, Rhs>(*this, b.derived()); + } +#else template<typename Rhs> inline const internal::solve_retval<LDLT, Rhs> solve(const MatrixBase<Rhs>& b) const @@ -183,6 +194,7 @@ template<typename _MatrixType, int _UpLo> class LDLT && "LDLT::solve(): invalid number of rows of the right hand side matrix b"); return internal::solve_retval<LDLT, Rhs>(*this, b.derived()); } +#endif template<typename Derived> bool solveInPlace(MatrixBase<Derived> &bAndX) const; @@ -217,6 +229,12 @@ template<typename _MatrixType, int _UpLo> class LDLT eigen_assert(m_isInitialized && "LDLT is not initialized."); return Success; } + + #ifndef EIGEN_PARSED_BY_DOXYGEN + template<typename RhsType, typename DstType> + EIGEN_DEVICE_FUNC + void _solve_impl(const RhsType &rhs, DstType &dst) const; + #endif protected: @@ -466,7 +484,49 @@ LDLT<MatrixType,_UpLo>& LDLT<MatrixType,_UpLo>::rankUpdate(const MatrixBase<Deri return *this; } +#ifndef EIGEN_PARSED_BY_DOXYGEN +template<typename _MatrixType, int _UpLo> +template<typename RhsType, typename DstType> +void LDLT<_MatrixType,_UpLo>::_solve_impl(const RhsType &rhs, DstType &dst) const +{ + eigen_assert(rhs.rows() == rows()); + // dst = P b + dst = m_transpositions * rhs; + + // dst = L^-1 (P b) + matrixL().solveInPlace(dst); + + // dst = D^-1 (L^-1 P b) + // more precisely, use pseudo-inverse of D (see bug 241) + using std::abs; + EIGEN_USING_STD_MATH(max); + const typename Diagonal<const MatrixType>::RealReturnType vecD(vectorD()); + // In some previous versions, tolerance was set to the max of 1/highest and the maximal diagonal entry * epsilon + // as motivated by LAPACK's xGELSS: + // RealScalar tolerance = (max)(vectorD.array().abs().maxCoeff() *NumTraits<RealScalar>::epsilon(),RealScalar(1) / NumTraits<RealScalar>::highest()); + // However, LDLT is not rank revealing, and so adjusting the tolerance wrt to the highest + // diagonal element is not well justified and to numerical issues in some cases. + // Moreover, Lapack's xSYTRS routines use 0 for the tolerance. + RealScalar tolerance = RealScalar(1) / NumTraits<RealScalar>::highest(); + + for (Index i = 0; i < vecD.size(); ++i) + { + if(abs(vecD(i)) > tolerance) + dst.row(i) /= vecD(i); + else + dst.row(i).setZero(); + } + + // dst = L^-T (D^-1 L^-1 P b) + matrixU().solveInPlace(dst); + + // dst = P^-1 (L^-T D^-1 L^-1 P b) = A^-1 b + dst = m_transpositions.transpose() * dst; +} +#endif + namespace internal { +#ifndef EIGEN_TEST_EVALUATORS template<typename _MatrixType, int _UpLo, typename Rhs> struct solve_retval<LDLT<_MatrixType,_UpLo>, Rhs> : solve_retval_base<LDLT<_MatrixType,_UpLo>, Rhs> @@ -476,41 +536,10 @@ struct solve_retval<LDLT<_MatrixType,_UpLo>, Rhs> template<typename Dest> void evalTo(Dest& dst) const { - eigen_assert(rhs().rows() == dec().matrixLDLT().rows()); - // dst = P b - dst = dec().transpositionsP() * rhs(); - - // dst = L^-1 (P b) - dec().matrixL().solveInPlace(dst); - - // dst = D^-1 (L^-1 P b) - // more precisely, use pseudo-inverse of D (see bug 241) - using std::abs; - EIGEN_USING_STD_MATH(max); - typedef typename LDLTType::MatrixType MatrixType; - typedef typename LDLTType::RealScalar RealScalar; - const typename Diagonal<const MatrixType>::RealReturnType vectorD(dec().vectorD()); - // In some previous versions, tolerance was set to the max of 1/highest and the maximal diagonal entry * epsilon - // as motivated by LAPACK's xGELSS: - // RealScalar tolerance = (max)(vectorD.array().abs().maxCoeff() *NumTraits<RealScalar>::epsilon(),RealScalar(1) / NumTraits<RealScalar>::highest()); - // However, LDLT is not rank revealing, and so adjusting the tolerance wrt to the highest - // diagonal element is not well justified and to numerical issues in some cases. - // Moreover, Lapack's xSYTRS routines use 0 for the tolerance. - RealScalar tolerance = RealScalar(1) / NumTraits<RealScalar>::highest(); - for (Index i = 0; i < vectorD.size(); ++i) { - if(abs(vectorD(i)) > tolerance) - dst.row(i) /= vectorD(i); - else - dst.row(i).setZero(); - } - - // dst = L^-T (D^-1 L^-1 P b) - dec().matrixU().solveInPlace(dst); - - // dst = P^-1 (L^-T D^-1 L^-1 P b) = A^-1 b - dst = dec().transpositionsP().transpose() * dst; + dec()._solve_impl(rhs(),dst); } }; +#endif } /** \internal use x = ldlt_object.solve(x); diff --git a/Eigen/src/Cholesky/LLT.h b/Eigen/src/Cholesky/LLT.h index 38e820165..89fb9a011 100644 --- a/Eigen/src/Cholesky/LLT.h +++ b/Eigen/src/Cholesky/LLT.h @@ -117,6 +117,17 @@ template<typename _MatrixType, int _UpLo> class LLT * * \sa solveInPlace(), MatrixBase::llt(), SelfAdjointView::llt() */ +#ifdef EIGEN_TEST_EVALUATORS + template<typename Rhs> + inline const Solve<LLT, Rhs> + solve(const MatrixBase<Rhs>& b) const + { + eigen_assert(m_isInitialized && "LLT is not initialized."); + eigen_assert(m_matrix.rows()==b.rows() + && "LLT::solve(): invalid number of rows of the right hand side matrix b"); + return Solve<LLT, Rhs>(*this, b.derived()); + } +#else template<typename Rhs> inline const internal::solve_retval<LLT, Rhs> solve(const MatrixBase<Rhs>& b) const @@ -126,6 +137,7 @@ template<typename _MatrixType, int _UpLo> class LLT && "LLT::solve(): invalid number of rows of the right hand side matrix b"); return internal::solve_retval<LLT, Rhs>(*this, b.derived()); } +#endif template<typename Derived> void solveInPlace(MatrixBase<Derived> &bAndX) const; @@ -161,6 +173,12 @@ template<typename _MatrixType, int _UpLo> class LLT template<typename VectorType> LLT rankUpdate(const VectorType& vec, const RealScalar& sigma = 1); + + #ifndef EIGEN_PARSED_BY_DOXYGEN + template<typename RhsType, typename DstType> + EIGEN_DEVICE_FUNC + void _solve_impl(const RhsType &rhs, DstType &dst) const; + #endif protected: /** \internal @@ -404,8 +422,19 @@ LLT<_MatrixType,_UpLo> LLT<_MatrixType,_UpLo>::rankUpdate(const VectorType& v, c return *this; } - + +#ifndef EIGEN_PARSED_BY_DOXYGEN +template<typename _MatrixType,int _UpLo> +template<typename RhsType, typename DstType> +void LLT<_MatrixType,_UpLo>::_solve_impl(const RhsType &rhs, DstType &dst) const +{ + dst = rhs; + solveInPlace(dst); +} +#endif + namespace internal { +#ifndef EIGEN_TEST_EVALUATORS template<typename _MatrixType, int UpLo, typename Rhs> struct solve_retval<LLT<_MatrixType, UpLo>, Rhs> : solve_retval_base<LLT<_MatrixType, UpLo>, Rhs> @@ -419,6 +448,7 @@ struct solve_retval<LLT<_MatrixType, UpLo>, Rhs> dec().solveInPlace(dst); } }; +#endif } /** \internal use x = llt_object.solve(x); diff --git a/Eigen/src/Core/ArrayBase.h b/Eigen/src/Core/ArrayBase.h index 2c9ace4a7..f5bae6357 100644 --- a/Eigen/src/Core/ArrayBase.h +++ b/Eigen/src/Core/ArrayBase.h @@ -64,8 +64,10 @@ template<typename Derived> class ArrayBase using Base::MaxSizeAtCompileTime; using Base::IsVectorAtCompileTime; using Base::Flags; +#ifndef EIGEN_TEST_EVALUATORS using Base::CoeffReadCost; - +#endif + using Base::derived; using Base::const_cast_derived; using Base::rows; @@ -121,7 +123,11 @@ template<typename Derived> class ArrayBase EIGEN_DEVICE_FUNC Derived& operator=(const ArrayBase& other) { +#ifndef EIGEN_TEST_EVALUATORS return internal::assign_selector<Derived,Derived>::run(derived(), other.derived()); +#else + internal::call_assignment(derived(), other.derived()); +#endif } EIGEN_DEVICE_FUNC @@ -177,6 +183,59 @@ template<typename Derived> class ArrayBase {EIGEN_STATIC_ASSERT(std::ptrdiff_t(sizeof(typename OtherDerived::Scalar))==-1,YOU_CANNOT_MIX_ARRAYS_AND_MATRICES); return *this;} }; +#ifdef EIGEN_TEST_EVALUATORS +/** replaces \c *this by \c *this - \a other. + * + * \returns a reference to \c *this + */ +template<typename Derived> +template<typename OtherDerived> +EIGEN_STRONG_INLINE Derived & +ArrayBase<Derived>::operator-=(const ArrayBase<OtherDerived> &other) +{ + call_assignment(derived(), other.derived(), internal::sub_assign_op<Scalar>()); + return derived(); +} + +/** replaces \c *this by \c *this + \a other. + * + * \returns a reference to \c *this + */ +template<typename Derived> +template<typename OtherDerived> +EIGEN_STRONG_INLINE Derived & +ArrayBase<Derived>::operator+=(const ArrayBase<OtherDerived>& other) +{ + call_assignment(derived(), other.derived(), internal::add_assign_op<Scalar>()); + return derived(); +} + +/** replaces \c *this by \c *this * \a other coefficient wise. + * + * \returns a reference to \c *this + */ +template<typename Derived> +template<typename OtherDerived> +EIGEN_STRONG_INLINE Derived & +ArrayBase<Derived>::operator*=(const ArrayBase<OtherDerived>& other) +{ + call_assignment(derived(), other.derived(), internal::mul_assign_op<Scalar,typename OtherDerived::Scalar>()); + return derived(); +} + +/** replaces \c *this by \c *this / \a other coefficient wise. + * + * \returns a reference to \c *this + */ +template<typename Derived> +template<typename OtherDerived> +EIGEN_STRONG_INLINE Derived & +ArrayBase<Derived>::operator/=(const ArrayBase<OtherDerived>& other) +{ + call_assignment(derived(), other.derived(), internal::div_assign_op<Scalar>()); + return derived(); +} +#else // EIGEN_TEST_EVALUATORS /** replaces \c *this by \c *this - \a other. * * \returns a reference to \c *this @@ -232,6 +291,7 @@ ArrayBase<Derived>::operator/=(const ArrayBase<OtherDerived>& other) tmp = other.derived(); return derived(); } +#endif } // end namespace Eigen diff --git a/Eigen/src/Core/ArrayWrapper.h b/Eigen/src/Core/ArrayWrapper.h index 4bb648024..599d87f64 100644 --- a/Eigen/src/Core/ArrayWrapper.h +++ b/Eigen/src/Core/ArrayWrapper.h @@ -39,6 +39,7 @@ class ArrayWrapper : public ArrayBase<ArrayWrapper<ExpressionType> > typedef ArrayBase<ArrayWrapper> Base; EIGEN_DENSE_PUBLIC_INTERFACE(ArrayWrapper) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(ArrayWrapper) + typedef typename internal::remove_all<ExpressionType>::type NestedExpression; typedef typename internal::conditional< internal::is_lvalue<ExpressionType>::value, @@ -176,6 +177,7 @@ class MatrixWrapper : public MatrixBase<MatrixWrapper<ExpressionType> > typedef MatrixBase<MatrixWrapper<ExpressionType> > Base; EIGEN_DENSE_PUBLIC_INTERFACE(MatrixWrapper) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(MatrixWrapper) + typedef typename internal::remove_all<ExpressionType>::type NestedExpression; typedef typename internal::conditional< internal::is_lvalue<ExpressionType>::value, diff --git a/Eigen/src/Core/Assign.h b/Eigen/src/Core/Assign.h index 07da2fe31..071cfbf7e 100644 --- a/Eigen/src/Core/Assign.h +++ b/Eigen/src/Core/Assign.h @@ -13,7 +13,7 @@ #define EIGEN_ASSIGN_H namespace Eigen { - +#ifndef EIGEN_TEST_EVALUATORS namespace internal { /*************************************************************************** @@ -489,6 +489,8 @@ struct assign_impl<Derived1, Derived2, SliceVectorizedTraversal, NoUnrolling, Ve } // end namespace internal +#endif // EIGEN_TEST_EVALUATORS + /*************************************************************************** * Part 4 : implementation of DenseBase methods ***************************************************************************/ @@ -508,11 +510,8 @@ EIGEN_STRONG_INLINE Derived& DenseBase<Derived> #ifdef EIGEN_TEST_EVALUATORS -#ifdef EIGEN_DEBUG_ASSIGN - internal::copy_using_evaluator_traits<Derived, OtherDerived>::debug(); -#endif eigen_assert(rows() == other.rows() && cols() == other.cols()); - internal::call_dense_assignment_loop(derived(),other.derived()); + internal::call_assignment_no_alias(derived(),other.derived()); #else // EIGEN_TEST_EVALUATORS @@ -533,6 +532,7 @@ EIGEN_STRONG_INLINE Derived& DenseBase<Derived> namespace internal { +#ifndef EIGEN_TEST_EVALUATORS template<typename Derived, typename OtherDerived, bool EvalBeforeAssigning = (int(internal::traits<OtherDerived>::Flags) & EvalBeforeAssigningBit) != 0, bool NeedToTranspose = ((int(Derived::RowsAtCompileTime) == 1 && int(OtherDerived::ColsAtCompileTime) == 1) @@ -568,9 +568,62 @@ struct assign_selector<Derived,OtherDerived,true,true> { EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Derived& run(Derived& dst, const OtherDerived& other) { return dst.lazyAssign(other.transpose().eval()); } }; - +#endif // EIGEN_TEST_EVALUATORS } // end namespace internal +#ifdef EIGEN_TEST_EVALUATORS +template<typename Derived> +template<typename OtherDerived> +EIGEN_DEVICE_FUNC +EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::operator=(const DenseBase<OtherDerived>& other) +{ + internal::call_assignment(derived(), other.derived()); + return derived(); +} + +template<typename Derived> +EIGEN_DEVICE_FUNC +EIGEN_STRONG_INLINE Derived& DenseBase<Derived>::operator=(const DenseBase& other) +{ + internal::call_assignment(derived(), other.derived()); + return derived(); +} + +template<typename Derived> +EIGEN_DEVICE_FUNC +EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const MatrixBase& other) +{ + internal::call_assignment(derived(), other.derived()); + return derived(); +} + +template<typename Derived> +template <typename OtherDerived> +EIGEN_DEVICE_FUNC +EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const DenseBase<OtherDerived>& other) +{ + internal::call_assignment(derived(), other.derived()); + return derived(); +} + +template<typename Derived> +template <typename OtherDerived> +EIGEN_DEVICE_FUNC +EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const EigenBase<OtherDerived>& other) +{ + internal::call_assignment(derived(), other.derived()); + return derived(); +} + +template<typename Derived> +template<typename OtherDerived> +EIGEN_DEVICE_FUNC +EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const ReturnByValue<OtherDerived>& other) +{ + other.derived().evalTo(derived()); + return derived(); +} +#else // EIGEN_TEST_EVALUATORS template<typename Derived> template<typename OtherDerived> EIGEN_DEVICE_FUNC @@ -616,6 +669,7 @@ EIGEN_STRONG_INLINE Derived& MatrixBase<Derived>::operator=(const ReturnByValue< { return internal::assign_selector<Derived,OtherDerived,false>::evalTo(derived(), other.derived()); } +#endif // EIGEN_TEST_EVALUATORS } // end namespace Eigen diff --git a/Eigen/src/Core/AssignEvaluator.h b/Eigen/src/Core/AssignEvaluator.h index 5451a138f..1727a6a4a 100644 --- a/Eigen/src/Core/AssignEvaluator.h +++ b/Eigen/src/Core/AssignEvaluator.h @@ -2,7 +2,7 @@ // for linear algebra. // // Copyright (C) 2011 Benoit Jacob <jacob.benoit.1@gmail.com> -// Copyright (C) 2011-2013 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2011-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // Copyright (C) 2011-2012 Jitse Niesen <jitse@maths.leeds.ac.uk> // // This Source Code Form is subject to the terms of the Mozilla @@ -24,37 +24,46 @@ namespace internal { // copy_using_evaluator_traits is based on assign_traits -template <typename Derived, typename OtherDerived> +template <typename DstEvaluator, typename SrcEvaluator, typename AssignFunc> struct copy_using_evaluator_traits { + typedef typename DstEvaluator::XprType Dst; + + enum { + DstFlags = DstEvaluator::Flags, + SrcFlags = SrcEvaluator::Flags + }; + public: enum { - DstIsAligned = Derived::Flags & AlignedBit, - DstHasDirectAccess = Derived::Flags & DirectAccessBit, - SrcIsAligned = OtherDerived::Flags & AlignedBit, - JointAlignment = bool(DstIsAligned) && bool(SrcIsAligned) ? Aligned : Unaligned, - SrcEvalBeforeAssign = (evaluator_traits<OtherDerived>::HasEvalTo == 1) + DstIsAligned = DstFlags & AlignedBit, + DstHasDirectAccess = DstFlags & DirectAccessBit, + SrcIsAligned = SrcFlags & AlignedBit, + JointAlignment = bool(DstIsAligned) && bool(SrcIsAligned) ? Aligned : Unaligned }; private: enum { - InnerSize = int(Derived::IsVectorAtCompileTime) ? int(Derived::SizeAtCompileTime) - : int(Derived::Flags)&RowMajorBit ? int(Derived::ColsAtCompileTime) - : int(Derived::RowsAtCompileTime), - InnerMaxSize = int(Derived::IsVectorAtCompileTime) ? int(Derived::MaxSizeAtCompileTime) - : int(Derived::Flags)&RowMajorBit ? int(Derived::MaxColsAtCompileTime) - : int(Derived::MaxRowsAtCompileTime), - MaxSizeAtCompileTime = Derived::SizeAtCompileTime, - PacketSize = packet_traits<typename Derived::Scalar>::size + InnerSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::SizeAtCompileTime) + : int(DstFlags)&RowMajorBit ? int(Dst::ColsAtCompileTime) + : int(Dst::RowsAtCompileTime), + InnerMaxSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::MaxSizeAtCompileTime) + : int(DstFlags)&RowMajorBit ? int(Dst::MaxColsAtCompileTime) + : int(Dst::MaxRowsAtCompileTime), + MaxSizeAtCompileTime = Dst::SizeAtCompileTime, + PacketSize = packet_traits<typename Dst::Scalar>::size }; enum { - StorageOrdersAgree = (int(Derived::IsRowMajor) == int(OtherDerived::IsRowMajor)), + DstIsRowMajor = DstEvaluator::Flags&RowMajorBit, + SrcIsRowMajor = SrcEvaluator::Flags&RowMajorBit, + StorageOrdersAgree = (int(DstIsRowMajor) == int(SrcIsRowMajor)), MightVectorize = StorageOrdersAgree - && (int(Derived::Flags) & int(OtherDerived::Flags) & ActualPacketAccessBit), + && (int(DstFlags) & int(SrcFlags) & ActualPacketAccessBit) + && (functor_traits<AssignFunc>::PacketAccess), MayInnerVectorize = MightVectorize && int(InnerSize)!=Dynamic && int(InnerSize)%int(PacketSize)==0 && int(DstIsAligned) && int(SrcIsAligned), - MayLinearize = StorageOrdersAgree && (int(Derived::Flags) & int(OtherDerived::Flags) & LinearAccessBit), + MayLinearize = StorageOrdersAgree && (int(DstFlags) & int(SrcFlags) & LinearAccessBit), MayLinearVectorize = MightVectorize && MayLinearize && DstHasDirectAccess && (DstIsAligned || MaxSizeAtCompileTime == Dynamic), /* If the destination isn't aligned, we have to do runtime checks and we don't unroll, @@ -68,8 +77,7 @@ private: public: enum { - Traversal = int(SrcEvalBeforeAssign) ? int(AllAtOnceTraversal) - : int(MayInnerVectorize) ? int(InnerVectorizedTraversal) + Traversal = int(MayInnerVectorize) ? int(InnerVectorizedTraversal) : int(MayLinearVectorize) ? int(LinearVectorizedTraversal) : int(MaySliceVectorize) ? int(SliceVectorizedTraversal) : int(MayLinearize) ? int(LinearTraversal) @@ -82,12 +90,12 @@ public: private: enum { UnrollingLimit = EIGEN_UNROLLING_LIMIT * (Vectorized ? int(PacketSize) : 1), - MayUnrollCompletely = int(Derived::SizeAtCompileTime) != Dynamic - && int(OtherDerived::CoeffReadCost) != Dynamic - && int(Derived::SizeAtCompileTime) * int(OtherDerived::CoeffReadCost) <= int(UnrollingLimit), + MayUnrollCompletely = int(Dst::SizeAtCompileTime) != Dynamic + && int(SrcEvaluator::CoeffReadCost) != Dynamic + && int(Dst::SizeAtCompileTime) * int(SrcEvaluator::CoeffReadCost) <= int(UnrollingLimit), MayUnrollInner = int(InnerSize) != Dynamic - && int(OtherDerived::CoeffReadCost) != Dynamic - && int(InnerSize) * int(OtherDerived::CoeffReadCost) <= int(UnrollingLimit) + && int(SrcEvaluator::CoeffReadCost) != Dynamic + && int(InnerSize) * int(SrcEvaluator::CoeffReadCost) <= int(UnrollingLimit) }; public: @@ -110,6 +118,12 @@ public: #ifdef EIGEN_DEBUG_ASSIGN static void debug() { + std::cerr << "DstXpr: " << typeid(typename DstEvaluator::XprType).name() << std::endl; + std::cerr << "SrcXpr: " << typeid(typename SrcEvaluator::XprType).name() << std::endl; + std::cerr.setf(std::ios::hex, std::ios::basefield); + EIGEN_DEBUG_VAR(DstFlags) + EIGEN_DEBUG_VAR(SrcFlags) + std::cerr.unsetf(std::ios::hex); EIGEN_DEBUG_VAR(DstIsAligned) EIGEN_DEBUG_VAR(SrcIsAligned) EIGEN_DEBUG_VAR(JointAlignment) @@ -127,6 +141,7 @@ public: EIGEN_DEBUG_VAR(MayUnrollCompletely) EIGEN_DEBUG_VAR(MayUnrollInner) EIGEN_DEBUG_VAR(Unrolling) + std::cerr << std::endl; } #endif }; @@ -142,6 +157,7 @@ public: template<typename Kernel, int Index, int Stop> struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling { + // FIXME: this is not very clean, perhaps this information should be provided by the kernel? typedef typename Kernel::DstEvaluatorType DstEvaluatorType; typedef typename DstEvaluatorType::XprType DstXprType; @@ -206,9 +222,10 @@ struct copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, Stop, Stop template<typename Kernel, int Index, int Stop> struct copy_using_evaluator_innervec_CompleteUnrolling { + // FIXME: this is not very clean, perhaps this information should be provided by the kernel? typedef typename Kernel::DstEvaluatorType DstEvaluatorType; typedef typename DstEvaluatorType::XprType DstXprType; - + enum { outer = Index / DstXprType::InnerSizeAtCompileTime, inner = Index % DstXprType::InnerSizeAtCompileTime, @@ -235,8 +252,7 @@ struct copy_using_evaluator_innervec_InnerUnrolling static EIGEN_STRONG_INLINE void run(Kernel &kernel, int outer) { kernel.template assignPacketByOuterInner<Aligned, Aligned>(outer, Index); - typedef typename Kernel::DstEvaluatorType::XprType DstXprType; - enum { NextIndex = Index + packet_traits<typename DstXprType::Scalar>::size }; + enum { NextIndex = Index + packet_traits<typename Kernel::Scalar>::size }; copy_using_evaluator_innervec_InnerUnrolling<Kernel, NextIndex, Stop>::run(kernel, outer); } }; @@ -496,25 +512,8 @@ struct dense_assignment_loop<Kernel, SliceVectorizedTraversal, NoUnrolling> } }; -/**************************** -*** All-at-once traversal *** -****************************/ - -// TODO: this 'AllAtOnceTraversal' should be dropped or caught earlier (Gael) -// Indeed, what to do with the kernel's functor?? -template<typename Kernel> -struct dense_assignment_loop<Kernel, AllAtOnceTraversal, NoUnrolling> -{ - static inline void run(Kernel & kernel) - { - // Evaluate rhs in temporary to prevent aliasing problems in a = a * a; - // TODO: Do not pass the xpr object to evalTo() (Jitse) - kernel.srcEvaluator().evalTo(kernel.dstEvaluator(), kernel.dstExpression()); - } -}; - /*************************************************************************** -* Part 4 : Generic Assignment routine +* Part 4 : Generic dense assignment kernel ***************************************************************************/ // This class generalize the assignment of a coefficient (or packet) from one dense evaluator @@ -523,7 +522,7 @@ struct dense_assignment_loop<Kernel, AllAtOnceTraversal, NoUnrolling> // This abstraction level permits to keep the evaluation loops as simple and as generic as possible. // One can customize the assignment using this generic dense_assignment_kernel with different // functors, or by completely overloading it, by-passing a functor. -template<typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT, typename Functor> +template<typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT, typename Functor, int Version = Specialized> class generic_dense_assignment_kernel { protected: @@ -535,16 +534,22 @@ public: typedef SrcEvaluatorTypeT SrcEvaluatorType; typedef typename DstEvaluatorType::Scalar Scalar; typedef typename DstEvaluatorType::Index Index; - typedef copy_using_evaluator_traits<DstXprType, SrcXprType> AssignmentTraits; + typedef copy_using_evaluator_traits<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor> AssignmentTraits; generic_dense_assignment_kernel(DstEvaluatorType &dst, const SrcEvaluatorType &src, const Functor &func, DstXprType& dstExpr) : m_dst(dst), m_src(src), m_functor(func), m_dstExpr(dstExpr) - {} + { + #ifdef EIGEN_DEBUG_ASSIGN + AssignmentTraits::debug(); + #endif + } Index size() const { return m_dstExpr.size(); } Index innerSize() const { return m_dstExpr.innerSize(); } Index outerSize() const { return m_dstExpr.outerSize(); } + Index rows() const { return m_dstExpr.rows(); } + Index cols() const { return m_dstExpr.cols(); } Index outerStride() const { return m_dstExpr.outerStride(); } // TODO get rid of this one: @@ -553,16 +558,19 @@ public: DstEvaluatorType& dstEvaluator() { return m_dst; } const SrcEvaluatorType& srcEvaluator() const { return m_src; } + /// Assign src(row,col) to dst(row,col) through the assignment functor. void assignCoeff(Index row, Index col) { m_functor.assignCoeff(m_dst.coeffRef(row,col), m_src.coeff(row,col)); } + /// \sa assignCoeff(Index,Index) void assignCoeff(Index index) { m_functor.assignCoeff(m_dst.coeffRef(index), m_src.coeff(index)); } + /// \sa assignCoeff(Index,Index) void assignCoeffByOuterInner(Index outer, Index inner) { Index row = rowIndexByOuterInner(outer, inner); @@ -596,7 +604,7 @@ public: typedef typename DstEvaluatorType::ExpressionTraits Traits; return int(Traits::RowsAtCompileTime) == 1 ? 0 : int(Traits::ColsAtCompileTime) == 1 ? inner - : int(Traits::Flags)&RowMajorBit ? outer + : int(DstEvaluatorType::Flags)&RowMajorBit ? outer : inner; } @@ -605,7 +613,7 @@ public: typedef typename DstEvaluatorType::ExpressionTraits Traits; return int(Traits::ColsAtCompileTime) == 1 ? 0 : int(Traits::RowsAtCompileTime) == 1 ? inner - : int(Traits::Flags)&RowMajorBit ? inner + : int(DstEvaluatorType::Flags)&RowMajorBit ? inner : outer; } @@ -617,13 +625,13 @@ protected: DstXprType& m_dstExpr; }; +/*************************************************************************** +* Part 5 : Entry point for dense rectangular assignment +***************************************************************************/ + template<typename DstXprType, typename SrcXprType, typename Functor> void call_dense_assignment_loop(const DstXprType& dst, const SrcXprType& src, const Functor &func) { -#ifdef EIGEN_DEBUG_ASSIGN - // TODO these traits should be computed from information provided by the evaluators - internal::copy_using_evaluator_traits<DstXprType, SrcXprType>::debug(); -#endif eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); typedef typename evaluator<DstXprType>::type DstEvaluatorType; @@ -645,195 +653,140 @@ void call_dense_assignment_loop(const DstXprType& dst, const SrcXprType& src) } /*************************************************************************** -* Part 5 : Entry points +* Part 6 : Generic assignment ***************************************************************************/ -// Based on DenseBase::LazyAssign() -// The following functions are just for testing and they are meant to be moved to operator= and the likes. +// Based on the respective shapes of the destination and source, +// the class AssignmentKind determine the kind of assignment mechanism. +// AssignmentKind must define a Kind typedef. +template<typename DstShape, typename SrcShape> struct AssignmentKind; -template<typename DstXprType, template <typename> class StorageBase, typename SrcXprType> -EIGEN_STRONG_INLINE -const DstXprType& copy_using_evaluator(const NoAlias<DstXprType, StorageBase>& dst, - const EigenBase<SrcXprType>& src) -{ - return noalias_copy_using_evaluator(dst.expression(), src.derived(), internal::assign_op<typename DstXprType::Scalar>()); -} +// Assignement kind defined in this file: +struct Dense2Dense {}; +struct EigenBase2EigenBase {}; -template<typename XprType, int AssumeAliasing = evaluator_traits<XprType>::AssumeAliasing> -struct AddEvalIfAssumingAliasing; +template<typename,typename> struct AssignmentKind { typedef EigenBase2EigenBase Kind; }; +template<> struct AssignmentKind<DenseShape,DenseShape> { typedef Dense2Dense Kind; }; + +// This is the main assignment class +template< typename DstXprType, typename SrcXprType, typename Functor, + typename Kind = typename AssignmentKind< typename evaluator_traits<DstXprType>::Shape , typename evaluator_traits<SrcXprType>::Shape >::Kind, + typename Scalar = typename DstXprType::Scalar> +struct Assignment; -template<typename XprType> -struct AddEvalIfAssumingAliasing<XprType, 0> -{ - static const XprType& run(const XprType& xpr) - { - return xpr; - } -}; -template<typename XprType> -struct AddEvalIfAssumingAliasing<XprType, 1> -{ - static const EvalToTemp<XprType> run(const XprType& xpr) - { - return EvalToTemp<XprType>(xpr); - } -}; +// The only purpose of this call_assignment() function is to deal with noalias() / AssumeAliasing and automatic transposition. +// Indeed, I (Gael) think that this concept of AssumeAliasing was a mistake, and it makes thing quite complicated. +// So this intermediate function removes everything related to AssumeAliasing such that Assignment +// does not has to bother about these annoying details. -template<typename DstXprType, typename SrcXprType, typename Functor> -EIGEN_STRONG_INLINE -const DstXprType& copy_using_evaluator(const EigenBase<DstXprType>& dst, const EigenBase<SrcXprType>& src, const Functor &func) +template<typename Dst, typename Src> +void call_assignment(Dst& dst, const Src& src) { - return noalias_copy_using_evaluator(dst.const_cast_derived(), - AddEvalIfAssumingAliasing<SrcXprType>::run(src.derived()), - func - ); + call_assignment(dst, src, internal::assign_op<typename Dst::Scalar>()); } - -// this mimics operator= -template<typename DstXprType, typename SrcXprType> -EIGEN_STRONG_INLINE -const DstXprType& copy_using_evaluator(const EigenBase<DstXprType>& dst, const EigenBase<SrcXprType>& src) +template<typename Dst, typename Src> +void call_assignment(const Dst& dst, const Src& src) { - return copy_using_evaluator(dst.const_cast_derived(), src.derived(), internal::assign_op<typename DstXprType::Scalar>()); + call_assignment(dst, src, internal::assign_op<typename Dst::Scalar>()); } - -template<typename DstXprType, typename SrcXprType, typename Functor> -EIGEN_STRONG_INLINE -const DstXprType& noalias_copy_using_evaluator(const PlainObjectBase<DstXprType>& dst, const EigenBase<SrcXprType>& src, const Functor &func) + +// Deal with AssumeAliasing +template<typename Dst, typename Src, typename Func> +void call_assignment(Dst& dst, const Src& src, const Func& func, typename enable_if<evaluator_traits<Src>::AssumeAliasing==1, void*>::type = 0) { -#ifdef EIGEN_DEBUG_ASSIGN - internal::copy_using_evaluator_traits<DstXprType, SrcXprType>::debug(); -#endif -#ifdef EIGEN_NO_AUTOMATIC_RESIZING - eigen_assert((dst.size()==0 || (IsVectorAtCompileTime ? (dst.size() == src.size()) - : (dst.rows() == src.rows() && dst.cols() == src.cols()))) - && "Size mismatch. Automatic resizing is disabled because EIGEN_NO_AUTOMATIC_RESIZING is defined"); +#ifdef EIGEN_TEST_EVALUATORS + typename plain_matrix_type<Src>::type tmp(src); #else - dst.const_cast_derived().resizeLike(src.derived()); + typename Src::PlainObject tmp(src.rows(), src.cols()); + call_assignment_no_alias(tmp, src, internal::assign_op<typename Dst::Scalar>()); #endif - call_dense_assignment_loop(dst.const_cast_derived(), src.derived(), func); - return dst.derived(); + + call_assignment_no_alias(dst, tmp, func); } -template<typename DstXprType, typename SrcXprType, typename Functor> -EIGEN_STRONG_INLINE -const DstXprType& noalias_copy_using_evaluator(const EigenBase<DstXprType>& dst, const EigenBase<SrcXprType>& src, const Functor &func) +template<typename Dst, typename Src, typename Func> +void call_assignment(Dst& dst, const Src& src, const Func& func, typename enable_if<evaluator_traits<Src>::AssumeAliasing==0, void*>::type = 0) { - call_dense_assignment_loop(dst.const_cast_derived(), src.derived(), func); - return dst.derived(); + call_assignment_no_alias(dst, src, func); } -// Based on DenseBase::swap() -// TODO: Check whether we need to do something special for swapping two -// Arrays or Matrices. (Jitse) - -// Overload default assignPacket behavior for swapping them -template<typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT> -class swap_kernel : public generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, swap_assign_op<typename DstEvaluatorTypeT::Scalar> > +// by-pass AssumeAliasing +// FIXME the const version should probably not be needed +// When there is no aliasing, we require that 'dst' has been properly resized +template<typename Dst, template <typename> class StorageBase, typename Src, typename Func> +void call_assignment(const NoAlias<Dst,StorageBase>& dst, const Src& src, const Func& func) { - typedef generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, swap_assign_op<typename DstEvaluatorTypeT::Scalar> > Base; - typedef typename DstEvaluatorTypeT::PacketScalar PacketScalar; - using Base::m_dst; - using Base::m_src; - using Base::m_functor; - -public: - typedef typename Base::Scalar Scalar; - typedef typename Base::Index Index; - typedef typename Base::DstXprType DstXprType; - - swap_kernel(DstEvaluatorTypeT &dst, const SrcEvaluatorTypeT &src, DstXprType& dstExpr) - : Base(dst, src, swap_assign_op<Scalar>(), dstExpr) - {} - - template<int StoreMode, int LoadMode> - void assignPacket(Index row, Index col) - { - m_functor.template swapPacket<StoreMode,LoadMode,PacketScalar>(&m_dst.coeffRef(row,col), &const_cast<SrcEvaluatorTypeT&>(m_src).coeffRef(row,col)); - } - - template<int StoreMode, int LoadMode> - void assignPacket(Index index) - { - m_functor.template swapPacket<StoreMode,LoadMode,PacketScalar>(&m_dst.coeffRef(index), &const_cast<SrcEvaluatorTypeT&>(m_src).coeffRef(index)); - } - - // TODO find a simple way not to have to copy/paste this function from generic_dense_assignment_kernel, by simple I mean no CRTP (Gael) - template<int StoreMode, int LoadMode> - void assignPacketByOuterInner(Index outer, Index inner) - { - Index row = Base::rowIndexByOuterInner(outer, inner); - Index col = Base::colIndexByOuterInner(outer, inner); - assignPacket<StoreMode,LoadMode>(row, col); - } -}; - -template<typename DstXprType, typename SrcXprType> -void swap_using_evaluator(const DstXprType& dst, const SrcXprType& src) -{ - // TODO there is too much redundancy with call_dense_assignment_loop - - eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); - - typedef typename evaluator<DstXprType>::type DstEvaluatorType; - typedef typename evaluator<SrcXprType>::type SrcEvaluatorType; - - DstEvaluatorType dstEvaluator(dst); - SrcEvaluatorType srcEvaluator(src); - - typedef swap_kernel<DstEvaluatorType,SrcEvaluatorType> Kernel; - Kernel kernel(dstEvaluator, srcEvaluator, dst.const_cast_derived()); - - dense_assignment_loop<Kernel>::run(kernel); -} - -// Based on MatrixBase::operator+= (in CwiseBinaryOp.h) -template<typename DstXprType, typename SrcXprType> -void add_assign_using_evaluator(const MatrixBase<DstXprType>& dst, const MatrixBase<SrcXprType>& src) -{ - typedef typename DstXprType::Scalar Scalar; - copy_using_evaluator(dst.derived(), src.derived(), add_assign_op<Scalar>()); + call_assignment_no_alias(dst.expression(), src, func); } - -// Based on ArrayBase::operator+= -template<typename DstXprType, typename SrcXprType> -void add_assign_using_evaluator(const ArrayBase<DstXprType>& dst, const ArrayBase<SrcXprType>& src) +template<typename Dst, template <typename> class StorageBase, typename Src, typename Func> +void call_assignment(NoAlias<Dst,StorageBase>& dst, const Src& src, const Func& func) { - typedef typename DstXprType::Scalar Scalar; - copy_using_evaluator(dst.derived(), src.derived(), add_assign_op<Scalar>()); + call_assignment_no_alias(dst.expression(), src, func); } -// TODO: Add add_assign_using_evaluator for EigenBase ? (Jitse) -template<typename DstXprType, typename SrcXprType> -void subtract_assign_using_evaluator(const MatrixBase<DstXprType>& dst, const MatrixBase<SrcXprType>& src) +template<typename Dst, typename Src, typename Func> +void call_assignment_no_alias(Dst& dst, const Src& src, const Func& func) { - typedef typename DstXprType::Scalar Scalar; - copy_using_evaluator(dst.derived(), src.derived(), sub_assign_op<Scalar>()); -} + enum { + NeedToTranspose = ( (int(Dst::RowsAtCompileTime) == 1 && int(Src::ColsAtCompileTime) == 1) + | // FIXME | instead of || to please GCC 4.4.0 stupid warning "suggest parentheses around &&". + // revert to || as soon as not needed anymore. + (int(Dst::ColsAtCompileTime) == 1 && int(Src::RowsAtCompileTime) == 1)) + && int(Dst::SizeAtCompileTime) != 1 + }; -template<typename DstXprType, typename SrcXprType> -void subtract_assign_using_evaluator(const ArrayBase<DstXprType>& dst, const ArrayBase<SrcXprType>& src) -{ - typedef typename DstXprType::Scalar Scalar; - copy_using_evaluator(dst.derived(), src.derived(), sub_assign_op<Scalar>()); + typename Dst::Index dstRows = NeedToTranspose ? src.cols() : src.rows(); + typename Dst::Index dstCols = NeedToTranspose ? src.rows() : src.cols(); + if((dst.rows()!=dstRows) || (dst.cols()!=dstCols)) + dst.resize(dstRows, dstCols); + + typedef typename internal::conditional<NeedToTranspose, Transpose<Dst>, Dst>::type ActualDstTypeCleaned; + typedef typename internal::conditional<NeedToTranspose, Transpose<Dst>, Dst&>::type ActualDstType; + ActualDstType actualDst(dst); + + // TODO check whether this is the right place to perform these checks: + EIGEN_STATIC_ASSERT_LVALUE(Dst) + EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(ActualDstTypeCleaned,Src) + + // TODO this line is commented to allow matrix = permutation + // Actually, the "Scalar" type for a permutation matrix does not really make sense, + // perhaps it could be void, and EIGEN_CHECK_BINARY_COMPATIBILIY could allow micing void with anything...? +// EIGEN_CHECK_BINARY_COMPATIBILIY(Func,typename ActualDstTypeCleaned::Scalar,typename Src::Scalar); + + Assignment<ActualDstTypeCleaned,Src,Func>::run(actualDst, src, func); } - -template<typename DstXprType, typename SrcXprType> -void multiply_assign_using_evaluator(const ArrayBase<DstXprType>& dst, const ArrayBase<SrcXprType>& src) +template<typename Dst, typename Src> +void call_assignment_no_alias(Dst& dst, const Src& src) { - typedef typename DstXprType::Scalar Scalar; - copy_using_evaluator(dst.derived(), src.derived(), mul_assign_op<Scalar>()); + call_assignment_no_alias(dst, src, internal::assign_op<typename Dst::Scalar>()); } -template<typename DstXprType, typename SrcXprType> -void divide_assign_using_evaluator(const ArrayBase<DstXprType>& dst, const ArrayBase<SrcXprType>& src) +// Generic Dense to Dense assignment +template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> +struct Assignment<DstXprType, SrcXprType, Functor, Dense2Dense, Scalar> { - typedef typename DstXprType::Scalar Scalar; - copy_using_evaluator(dst.derived(), src.derived(), div_assign_op<Scalar>()); -} + static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) + { + eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); + + call_dense_assignment_loop(dst, src, func); + } +}; +// Generic assignment through evalTo. +// TODO: not sure we have to keep that one, but it helps porting current code to new evaluator mechanism. +template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> +struct Assignment<DstXprType, SrcXprType, Functor, EigenBase2EigenBase, Scalar> +{ + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar> &/*func*/) + { + eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); + + src.evalTo(dst); + } +}; } // namespace internal diff --git a/Eigen/src/Core/BandMatrix.h b/Eigen/src/Core/BandMatrix.h index ffd7fe8b3..ba4749707 100644 --- a/Eigen/src/Core/BandMatrix.h +++ b/Eigen/src/Core/BandMatrix.h @@ -327,6 +327,29 @@ class TridiagonalMatrix : public BandMatrix<Scalar,Size,Size,Options&SelfAdjoint protected: }; + +#ifdef EIGEN_TEST_EVALUATORS + +struct BandShape {}; + +template<typename _Scalar, int _Rows, int _Cols, int _Supers, int _Subs, int _Options> +struct evaluator_traits<BandMatrix<_Scalar,_Rows,_Cols,_Supers,_Subs,_Options> > + : public evaluator_traits_base<BandMatrix<_Scalar,_Rows,_Cols,_Supers,_Subs,_Options> > +{ + typedef BandShape Shape; +}; + +template<typename _CoefficientsType,int _Rows, int _Cols, int _Supers, int _Subs,int _Options> +struct evaluator_traits<BandMatrixWrapper<_CoefficientsType,_Rows,_Cols,_Supers,_Subs,_Options> > + : public evaluator_traits_base<BandMatrixWrapper<_CoefficientsType,_Rows,_Cols,_Supers,_Subs,_Options> > +{ + typedef BandShape Shape; +}; + +template<> struct AssignmentKind<DenseShape,BandShape> { typedef EigenBase2EigenBase Kind; }; +#endif + + } // end namespace internal } // end namespace Eigen diff --git a/Eigen/src/Core/Block.h b/Eigen/src/Core/Block.h index da193d1a2..0f3ab304a 100644 --- a/Eigen/src/Core/Block.h +++ b/Eigen/src/Core/Block.h @@ -68,6 +68,7 @@ struct traits<Block<XprType, BlockRows, BlockCols, InnerPanel> > : traits<XprTyp MaxColsAtCompileTime = BlockCols==0 ? 0 : ColsAtCompileTime != Dynamic ? int(ColsAtCompileTime) : int(traits<XprType>::MaxColsAtCompileTime), + XprTypeIsRowMajor = (int(traits<XprType>::Flags)&RowMajorBit) != 0, IsRowMajor = (MaxRowsAtCompileTime==1&&MaxColsAtCompileTime!=1) ? 1 : (MaxColsAtCompileTime==1&&MaxRowsAtCompileTime!=1) ? 0 @@ -80,6 +81,10 @@ struct traits<Block<XprType, BlockRows, BlockCols, InnerPanel> > : traits<XprTyp OuterStrideAtCompileTime = HasSameStorageOrderAsXprType ? int(outer_stride_at_compile_time<XprType>::ret) : int(inner_stride_at_compile_time<XprType>::ret), + // IsAligned is needed by MapBase's assertions + // We can sefely set it to false here. Internal alignment errors will be detected by an eigen_internal_assert in the respective evaluator + IsAligned = 0, +#ifndef EIGEN_TEST_EVALUATORS MaskPacketAccessBit = (InnerSize == Dynamic || (InnerSize % packet_traits<Scalar>::size) == 0) && (InnerStrideAtCompileTime == 1) ? PacketAccessBit : 0, @@ -92,6 +97,12 @@ struct traits<Block<XprType, BlockRows, BlockCols, InnerPanel> > : traits<XprTyp MaskPacketAccessBit | MaskAlignedBit), Flags = Flags0 | FlagsLinearAccessBit | FlagsLvalueBit | FlagsRowMajorBit +#else + // FIXME, this traits is rather specialized for dense object and it needs to be cleaned further + FlagsLvalueBit = is_lvalue<XprType>::value ? LvalueBit : 0, + FlagsRowMajorBit = IsRowMajor ? RowMajorBit : 0, + Flags = (traits<XprType>::Flags & DirectAccessBit) | FlagsLvalueBit | FlagsRowMajorBit // FIXME DirectAccessBit should not be handled by expressions +#endif }; }; @@ -111,6 +122,8 @@ template<typename XprType, int BlockRows, int BlockCols, bool InnerPanel> class typedef Impl Base; EIGEN_GENERIC_PUBLIC_INTERFACE(Block) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(Block) + + typedef typename internal::remove_all<XprType>::type NestedExpression; /** Column or Row constructor */ @@ -333,6 +346,9 @@ class BlockImpl_dense<XprType,BlockRows,BlockCols, InnerPanel,true> : public MapBase<Block<XprType, BlockRows, BlockCols, InnerPanel> > { typedef Block<XprType, BlockRows, BlockCols, InnerPanel> BlockType; + enum { + XprTypeIsRowMajor = (int(traits<XprType>::Flags)&RowMajorBit) != 0 + }; public: typedef MapBase<BlockType> Base; @@ -343,9 +359,8 @@ class BlockImpl_dense<XprType,BlockRows,BlockCols, InnerPanel,true> */ EIGEN_DEVICE_FUNC inline BlockImpl_dense(XprType& xpr, Index i) - : Base(internal::const_cast_ptr(&xpr.coeffRef( - (BlockRows==1) && (BlockCols==XprType::ColsAtCompileTime) ? i : 0, - (BlockRows==XprType::RowsAtCompileTime) && (BlockCols==1) ? i : 0)), + : Base(xpr.data() + i * ( ((BlockRows==1) && (BlockCols==XprType::ColsAtCompileTime) && (!XprTypeIsRowMajor)) + || ((BlockRows==XprType::RowsAtCompileTime) && (BlockCols==1) && ( XprTypeIsRowMajor)) ? xpr.innerStride() : xpr.outerStride()), BlockRows==1 ? 1 : xpr.rows(), BlockCols==1 ? 1 : xpr.cols()), m_xpr(xpr) @@ -357,7 +372,8 @@ class BlockImpl_dense<XprType,BlockRows,BlockCols, InnerPanel,true> */ EIGEN_DEVICE_FUNC inline BlockImpl_dense(XprType& xpr, Index startRow, Index startCol) - : Base(internal::const_cast_ptr(&xpr.coeffRef(startRow,startCol))), m_xpr(xpr) + : Base(xpr.data()+xpr.innerStride()*(XprTypeIsRowMajor?startCol:startRow) + xpr.outerStride()*(XprTypeIsRowMajor?startRow:startCol)), + m_xpr(xpr) { init(); } @@ -368,7 +384,7 @@ class BlockImpl_dense<XprType,BlockRows,BlockCols, InnerPanel,true> inline BlockImpl_dense(XprType& xpr, Index startRow, Index startCol, Index blockRows, Index blockCols) - : Base(internal::const_cast_ptr(&xpr.coeffRef(startRow,startCol)), blockRows, blockCols), + : Base(xpr.data()+xpr.innerStride()*(XprTypeIsRowMajor?startCol:startRow) + xpr.outerStride()*(XprTypeIsRowMajor?startRow:startCol), blockRows, blockCols), m_xpr(xpr) { init(); diff --git a/Eigen/src/Core/BooleanRedux.h b/Eigen/src/Core/BooleanRedux.h index be9f48a8c..192e1db53 100644 --- a/Eigen/src/Core/BooleanRedux.h +++ b/Eigen/src/Core/BooleanRedux.h @@ -17,10 +17,18 @@ namespace internal { template<typename Derived, int UnrollCount> struct all_unroller { +#ifdef EIGEN_TEST_EVALUATORS + typedef typename Derived::ExpressionTraits Traits; + enum { + col = (UnrollCount-1) / Traits::RowsAtCompileTime, + row = (UnrollCount-1) % Traits::RowsAtCompileTime + }; +#else enum { col = (UnrollCount-1) / Derived::RowsAtCompileTime, row = (UnrollCount-1) % Derived::RowsAtCompileTime }; +#endif static inline bool run(const Derived &mat) { @@ -43,11 +51,19 @@ struct all_unroller<Derived, Dynamic> template<typename Derived, int UnrollCount> struct any_unroller { +#ifdef EIGEN_TEST_EVALUATORS + typedef typename Derived::ExpressionTraits Traits; + enum { + col = (UnrollCount-1) / Traits::RowsAtCompileTime, + row = (UnrollCount-1) % Traits::RowsAtCompileTime + }; +#else enum { col = (UnrollCount-1) / Derived::RowsAtCompileTime, row = (UnrollCount-1) % Derived::RowsAtCompileTime }; - +#endif + static inline bool run(const Derived &mat) { return any_unroller<Derived, UnrollCount-1>::run(mat) || mat.coeff(row, col); @@ -78,12 +94,32 @@ struct any_unroller<Derived, Dynamic> template<typename Derived> inline bool DenseBase<Derived>::all() const { +#ifdef EIGEN_TEST_EVALUATORS + typedef typename internal::evaluator<Derived>::type Evaluator; + enum { + unroll = SizeAtCompileTime != Dynamic + && Evaluator::CoeffReadCost != Dynamic + && NumTraits<Scalar>::AddCost != Dynamic + && SizeAtCompileTime * (Evaluator::CoeffReadCost + NumTraits<Scalar>::AddCost) <= EIGEN_UNROLLING_LIMIT + }; + Evaluator evaluator(derived()); + if(unroll) + return internal::all_unroller<Evaluator, unroll ? int(SizeAtCompileTime) : Dynamic>::run(evaluator); + else + { + for(Index j = 0; j < cols(); ++j) + for(Index i = 0; i < rows(); ++i) + if (!evaluator.coeff(i, j)) return false; + return true; + } +#else enum { unroll = SizeAtCompileTime != Dynamic && CoeffReadCost != Dynamic && NumTraits<Scalar>::AddCost != Dynamic && SizeAtCompileTime * (CoeffReadCost + NumTraits<Scalar>::AddCost) <= EIGEN_UNROLLING_LIMIT }; + if(unroll) return internal::all_unroller<Derived, unroll ? int(SizeAtCompileTime) : Dynamic>::run(derived()); else @@ -93,6 +129,7 @@ inline bool DenseBase<Derived>::all() const if (!coeff(i, j)) return false; return true; } +#endif } /** \returns true if at least one coefficient is true @@ -102,6 +139,25 @@ inline bool DenseBase<Derived>::all() const template<typename Derived> inline bool DenseBase<Derived>::any() const { +#ifdef EIGEN_TEST_EVALUATORS + typedef typename internal::evaluator<Derived>::type Evaluator; + enum { + unroll = SizeAtCompileTime != Dynamic + && Evaluator::CoeffReadCost != Dynamic + && NumTraits<Scalar>::AddCost != Dynamic + && SizeAtCompileTime * (Evaluator::CoeffReadCost + NumTraits<Scalar>::AddCost) <= EIGEN_UNROLLING_LIMIT + }; + Evaluator evaluator(derived()); + if(unroll) + return internal::any_unroller<Evaluator, unroll ? int(SizeAtCompileTime) : Dynamic>::run(evaluator); + else + { + for(Index j = 0; j < cols(); ++j) + for(Index i = 0; i < rows(); ++i) + if (evaluator.coeff(i, j)) return true; + return false; + } +#else enum { unroll = SizeAtCompileTime != Dynamic && CoeffReadCost != Dynamic @@ -117,6 +173,7 @@ inline bool DenseBase<Derived>::any() const if (coeff(i, j)) return true; return false; } +#endif } /** \returns the number of coefficients which evaluate to true diff --git a/Eigen/src/Core/CoreEvaluators.h b/Eigen/src/Core/CoreEvaluators.h index 3568cb85f..8c9190a0e 100644 --- a/Eigen/src/Core/CoreEvaluators.h +++ b/Eigen/src/Core/CoreEvaluators.h @@ -2,7 +2,7 @@ // for linear algebra. // // Copyright (C) 2011 Benoit Jacob <jacob.benoit.1@gmail.com> -// Copyright (C) 2011 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2011-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // Copyright (C) 2011-2012 Jitse Niesen <jitse@maths.leeds.ac.uk> // // This Source Code Form is subject to the terms of the Mozilla @@ -14,57 +14,83 @@ #define EIGEN_COREEVALUATORS_H namespace Eigen { - + namespace internal { -// evaluator_traits<T> contains traits for evaluator_impl<T> +struct IndexBased {}; +struct IteratorBased {}; + +// This class returns the evaluator kind from the expression storage kind. +// Default assumes index based accessors +template<typename StorageKind> +struct storage_kind_to_evaluator_kind { + typedef IndexBased Kind; +}; + +// This class returns the evaluator shape from the expression storage kind. +// It can be Dense, Sparse, Triangular, Diagonal, SelfAdjoint, Band, etc. +template<typename StorageKind> struct storage_kind_to_shape; + + +template<> struct storage_kind_to_shape<Dense> { typedef DenseShape Shape; }; + +// Evaluators have to be specialized with respect to various criteria such as: +// - storage/structure/shape +// - scalar type +// - etc. +// Therefore, we need specialization of evaluator providing additional template arguments for each kind of evaluators. +// We currently distinguish the following kind of evaluators: +// - unary_evaluator for expressions taking only one arguments (CwiseUnaryOp, CwiseUnaryView, Transpose, MatrixWrapper, ArrayWrapper, Reverse, Replicate) +// - binary_evaluator for expression taking two arguments (CwiseBinaryOp) +// - product_evaluator for linear algebra products (Product); special case of binary_evaluator because it requires additional tags for dispatching. +// - mapbase_evaluator for Map, Block, Ref +// - block_evaluator for Block (special dispatching to a mapbase_evaluator or unary_evaluator) + +template< typename T, + typename LhsKind = typename evaluator_traits<typename T::Lhs>::Kind, + typename RhsKind = typename evaluator_traits<typename T::Rhs>::Kind, + typename LhsScalar = typename T::Lhs::Scalar, + typename RhsScalar = typename T::Rhs::Scalar> struct binary_evaluator; + +template< typename T, + typename Kind = typename evaluator_traits<typename T::NestedExpression>::Kind, + typename Scalar = typename T::Scalar> struct unary_evaluator; + +// evaluator_traits<T> contains traits for evaluator<T> template<typename T> -struct evaluator_traits +struct evaluator_traits_base { - // 1 if evaluator_impl<T>::evalTo() exists - // 0 if evaluator_impl<T> allows coefficient-based access - static const int HasEvalTo = 0; - + // TODO check whether these two indirections are really needed. + // Basically, if nobody overwrite type and nestedType, then, they can be dropped +// typedef evaluator<T> type; +// typedef evaluator<T> nestedType; + + // by default, get evaluator kind and shape from storage + typedef typename storage_kind_to_evaluator_kind<typename traits<T>::StorageKind>::Kind Kind; + typedef typename storage_kind_to_shape<typename traits<T>::StorageKind>::Shape Shape; + // 1 if assignment A = B assumes aliasing when B is of type T and thus B needs to be evaluated into a // temporary; 0 if not. static const int AssumeAliasing = 0; }; -// expression class for evaluating nested expression to a temporary - -template<typename ArgType> -class EvalToTemp; - -// evaluator<T>::type is type of evaluator for T -// evaluator<T>::nestedType is type of evaluator if T is nested inside another evaluator - -template<typename T> -struct evaluator_impl -{ }; - -template<typename T, int Nested = evaluator_traits<T>::HasEvalTo> -struct evaluator_nested_type; - +// Default evaluator traits template<typename T> -struct evaluator_nested_type<T, 0> +struct evaluator_traits : public evaluator_traits_base<T> { - typedef evaluator_impl<T> type; }; -template<typename T> -struct evaluator_nested_type<T, 1> -{ - typedef evaluator_impl<EvalToTemp<T> > type; -}; +// By default, we assume a unary expression: template<typename T> -struct evaluator +struct evaluator : public unary_evaluator<T> { - typedef evaluator_impl<T> type; - typedef typename evaluator_nested_type<T>::type nestedType; + typedef unary_evaluator<T> Base; + evaluator(const T& xpr) : Base(xpr) {} }; + // TODO: Think about const-correctness template<typename T> @@ -76,46 +102,58 @@ struct evaluator<const T> // TODO this class does not seem to be necessary anymore template<typename ExpressionType> -struct evaluator_impl_base +struct evaluator_base { - typedef typename ExpressionType::Index Index; +// typedef typename evaluator_traits<ExpressionType>::type type; +// typedef typename evaluator_traits<ExpressionType>::nestedType nestedType; + typedef evaluator<ExpressionType> type; + typedef evaluator<ExpressionType> nestedType; + + typedef typename traits<ExpressionType>::Index Index; // TODO that's not very nice to have to propagate all these traits. They are currently only needed to handle outer,inner indices. typedef traits<ExpressionType> ExpressionTraits; - - evaluator_impl<ExpressionType>& derived() - { - return *static_cast<evaluator_impl<ExpressionType>*>(this); - } }; // -------------------- Matrix and Array -------------------- // -// evaluator_impl<PlainObjectBase> is a common base class for the +// evaluator<PlainObjectBase> is a common base class for the // Matrix and Array evaluators. +// Here we directly specialize evaluator. This is not really a unary expression, and it is, by definition, dense, +// so no need for more sophisticated dispatching. template<typename Derived> -struct evaluator_impl<PlainObjectBase<Derived> > - : evaluator_impl_base<Derived> +struct evaluator<PlainObjectBase<Derived> > + : evaluator_base<Derived> { typedef PlainObjectBase<Derived> PlainObjectType; + typedef typename PlainObjectType::Index Index; + typedef typename PlainObjectType::Scalar Scalar; + typedef typename PlainObjectType::CoeffReturnType CoeffReturnType; + typedef typename PlainObjectType::PacketScalar PacketScalar; + typedef typename PlainObjectType::PacketReturnType PacketReturnType; enum { IsRowMajor = PlainObjectType::IsRowMajor, IsVectorAtCompileTime = PlainObjectType::IsVectorAtCompileTime, RowsAtCompileTime = PlainObjectType::RowsAtCompileTime, - ColsAtCompileTime = PlainObjectType::ColsAtCompileTime + ColsAtCompileTime = PlainObjectType::ColsAtCompileTime, + + CoeffReadCost = NumTraits<Scalar>::ReadCost, + Flags = compute_matrix_evaluator_flags< Scalar,Derived::RowsAtCompileTime,Derived::ColsAtCompileTime, + Derived::Options,Derived::MaxRowsAtCompileTime,Derived::MaxColsAtCompileTime>::ret }; - - evaluator_impl(const PlainObjectType& m) + + evaluator() + : m_data(0), + m_outerStride(IsVectorAtCompileTime ? 0 + : int(IsRowMajor) ? ColsAtCompileTime + : RowsAtCompileTime) + {} + + evaluator(const PlainObjectType& m) : m_data(m.data()), m_outerStride(IsVectorAtCompileTime ? 0 : m.outerStride()) { } - typedef typename PlainObjectType::Index Index; - typedef typename PlainObjectType::Scalar Scalar; - typedef typename PlainObjectType::CoeffReturnType CoeffReturnType; - typedef typename PlainObjectType::PacketScalar PacketScalar; - typedef typename PlainObjectType::PacketReturnType PacketReturnType; - CoeffReturnType coeff(Index row, Index col) const { if (IsRowMajor) @@ -184,153 +222,45 @@ protected: }; template<typename Scalar, int Rows, int Cols, int Options, int MaxRows, int MaxCols> -struct evaluator_impl<Matrix<Scalar, Rows, Cols, Options, MaxRows, MaxCols> > - : evaluator_impl<PlainObjectBase<Matrix<Scalar, Rows, Cols, Options, MaxRows, MaxCols> > > +struct evaluator<Matrix<Scalar, Rows, Cols, Options, MaxRows, MaxCols> > + : evaluator<PlainObjectBase<Matrix<Scalar, Rows, Cols, Options, MaxRows, MaxCols> > > { typedef Matrix<Scalar, Rows, Cols, Options, MaxRows, MaxCols> XprType; + + evaluator() {} - evaluator_impl(const XprType& m) - : evaluator_impl<PlainObjectBase<XprType> >(m) + evaluator(const XprType& m) + : evaluator<PlainObjectBase<XprType> >(m) { } }; template<typename Scalar, int Rows, int Cols, int Options, int MaxRows, int MaxCols> -struct evaluator_impl<Array<Scalar, Rows, Cols, Options, MaxRows, MaxCols> > - : evaluator_impl<PlainObjectBase<Array<Scalar, Rows, Cols, Options, MaxRows, MaxCols> > > +struct evaluator<Array<Scalar, Rows, Cols, Options, MaxRows, MaxCols> > + : evaluator<PlainObjectBase<Array<Scalar, Rows, Cols, Options, MaxRows, MaxCols> > > { typedef Array<Scalar, Rows, Cols, Options, MaxRows, MaxCols> XprType; - evaluator_impl(const XprType& m) - : evaluator_impl<PlainObjectBase<XprType> >(m) - { } -}; - -// -------------------- EvalToTemp -------------------- - -template<typename ArgType> -struct traits<EvalToTemp<ArgType> > - : public traits<ArgType> -{ }; - -template<typename ArgType> -class EvalToTemp - : public dense_xpr_base<EvalToTemp<ArgType> >::type -{ - public: - - typedef typename dense_xpr_base<EvalToTemp>::type Base; - EIGEN_GENERIC_PUBLIC_INTERFACE(EvalToTemp) - - EvalToTemp(const ArgType& arg) - : m_arg(arg) - { } - - const ArgType& arg() const - { - return m_arg; - } - - Index rows() const - { - return m_arg.rows(); - } - - Index cols() const - { - return m_arg.cols(); - } - - private: - const ArgType& m_arg; -}; - -template<typename ArgType> -struct evaluator_impl<EvalToTemp<ArgType> > -{ - typedef EvalToTemp<ArgType> XprType; - typedef typename ArgType::PlainObject PlainObject; - - evaluator_impl(const XprType& xpr) - : m_result(xpr.rows(), xpr.cols()), m_resultImpl(m_result) - { - // TODO we should simply do m_result(xpr.arg()); - call_dense_assignment_loop(m_result, xpr.arg()); - } - - // This constructor is used when nesting an EvalTo evaluator in another evaluator - evaluator_impl(const ArgType& arg) - : m_result(arg.rows(), arg.cols()), m_resultImpl(m_result) - { - // TODO we should simply do m_result(xpr.arg()); - call_dense_assignment_loop(m_result, arg); - } - - typedef typename PlainObject::Index Index; - typedef typename PlainObject::Scalar Scalar; - typedef typename PlainObject::CoeffReturnType CoeffReturnType; - typedef typename PlainObject::PacketScalar PacketScalar; - typedef typename PlainObject::PacketReturnType PacketReturnType; - - // All other functions are forwarded to m_resultImpl - - CoeffReturnType coeff(Index row, Index col) const - { - return m_resultImpl.coeff(row, col); - } - - CoeffReturnType coeff(Index index) const - { - return m_resultImpl.coeff(index); - } + evaluator() {} - Scalar& coeffRef(Index row, Index col) - { - return m_resultImpl.coeffRef(row, col); - } - - Scalar& coeffRef(Index index) - { - return m_resultImpl.coeffRef(index); - } - - template<int LoadMode> - PacketReturnType packet(Index row, Index col) const - { - return m_resultImpl.template packet<LoadMode>(row, col); - } - - template<int LoadMode> - PacketReturnType packet(Index index) const - { - return m_resultImpl.packet<LoadMode>(index); - } - - template<int StoreMode> - void writePacket(Index row, Index col, const PacketScalar& x) - { - m_resultImpl.template writePacket<StoreMode>(row, col, x); - } - - template<int StoreMode> - void writePacket(Index index, const PacketScalar& x) - { - m_resultImpl.template writePacket<StoreMode>(index, x); - } - -protected: - PlainObject m_result; - typename evaluator<PlainObject>::nestedType m_resultImpl; + evaluator(const XprType& m) + : evaluator<PlainObjectBase<XprType> >(m) + { } }; // -------------------- Transpose -------------------- template<typename ArgType> -struct evaluator_impl<Transpose<ArgType> > - : evaluator_impl_base<Transpose<ArgType> > +struct unary_evaluator<Transpose<ArgType>, IndexBased> + : evaluator_base<Transpose<ArgType> > { typedef Transpose<ArgType> XprType; + + enum { + CoeffReadCost = evaluator<ArgType>::CoeffReadCost, + Flags = evaluator<ArgType>::Flags ^ RowMajorBit + }; - evaluator_impl(const XprType& t) : m_argImpl(t.nestedExpression()) {} + unary_evaluator(const XprType& t) : m_argImpl(t.nestedExpression()) {} typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; @@ -387,13 +317,27 @@ protected: }; // -------------------- CwiseNullaryOp -------------------- +// Like Matrix and Array, this is not really a unary expression, so we directly specialize evaluator. +// Likewise, there is not need to more sophisticated dispatching here. template<typename NullaryOp, typename PlainObjectType> -struct evaluator_impl<CwiseNullaryOp<NullaryOp,PlainObjectType> > +struct evaluator<CwiseNullaryOp<NullaryOp,PlainObjectType> > + : evaluator_base<CwiseNullaryOp<NullaryOp,PlainObjectType> > { typedef CwiseNullaryOp<NullaryOp,PlainObjectType> XprType; + typedef typename internal::remove_all<PlainObjectType>::type PlainObjectTypeCleaned; + + enum { + CoeffReadCost = internal::functor_traits<NullaryOp>::Cost, + + Flags = (evaluator<PlainObjectTypeCleaned>::Flags + & ( HereditaryBits + | (functor_has_linear_access<NullaryOp>::ret ? LinearAccessBit : 0) + | (functor_traits<NullaryOp>::PacketAccess ? PacketAccessBit : 0))) + | (functor_traits<NullaryOp>::IsRepeatable ? 0 : EvalBeforeNestingBit) // FIXME EvalBeforeNestingBit should be needed anymore + }; - evaluator_impl(const XprType& n) + evaluator(const XprType& n) : m_functor(n.functor()) { } @@ -430,11 +374,20 @@ protected: // -------------------- CwiseUnaryOp -------------------- template<typename UnaryOp, typename ArgType> -struct evaluator_impl<CwiseUnaryOp<UnaryOp, ArgType> > +struct unary_evaluator<CwiseUnaryOp<UnaryOp, ArgType>, IndexBased > + : evaluator_base<CwiseUnaryOp<UnaryOp, ArgType> > { typedef CwiseUnaryOp<UnaryOp, ArgType> XprType; + + enum { + CoeffReadCost = evaluator<ArgType>::CoeffReadCost + functor_traits<UnaryOp>::Cost, + + Flags = evaluator<ArgType>::Flags & ( + HereditaryBits | LinearAccessBit | AlignedBit + | (functor_traits<UnaryOp>::PacketAccess ? PacketAccessBit : 0)) + }; - evaluator_impl(const XprType& op) + unary_evaluator(const XprType& op) : m_functor(op.functor()), m_argImpl(op.nestedExpression()) { } @@ -472,12 +425,43 @@ protected: // -------------------- CwiseBinaryOp -------------------- +// this is a binary expression template<typename BinaryOp, typename Lhs, typename Rhs> -struct evaluator_impl<CwiseBinaryOp<BinaryOp, Lhs, Rhs> > +struct evaluator<CwiseBinaryOp<BinaryOp, Lhs, Rhs> > + : public binary_evaluator<CwiseBinaryOp<BinaryOp, Lhs, Rhs> > { typedef CwiseBinaryOp<BinaryOp, Lhs, Rhs> XprType; + typedef binary_evaluator<CwiseBinaryOp<BinaryOp, Lhs, Rhs> > Base; + + evaluator(const XprType& xpr) : Base(xpr) {} +}; - evaluator_impl(const XprType& xpr) +template<typename BinaryOp, typename Lhs, typename Rhs> +struct binary_evaluator<CwiseBinaryOp<BinaryOp, Lhs, Rhs>, IndexBased, IndexBased> + : evaluator_base<CwiseBinaryOp<BinaryOp, Lhs, Rhs> > +{ + typedef CwiseBinaryOp<BinaryOp, Lhs, Rhs> XprType; + + enum { + CoeffReadCost = evaluator<Lhs>::CoeffReadCost + evaluator<Rhs>::CoeffReadCost + functor_traits<BinaryOp>::Cost, + + LhsFlags = evaluator<Lhs>::Flags, + RhsFlags = evaluator<Rhs>::Flags, + SameType = is_same<typename Lhs::Scalar,typename Rhs::Scalar>::value, + StorageOrdersAgree = (int(LhsFlags)&RowMajorBit)==(int(RhsFlags)&RowMajorBit), + Flags0 = (int(LhsFlags) | int(RhsFlags)) & ( + HereditaryBits + | (int(LhsFlags) & int(RhsFlags) & + ( AlignedBit + | (StorageOrdersAgree ? LinearAccessBit : 0) + | (functor_traits<BinaryOp>::PacketAccess && StorageOrdersAgree && SameType ? PacketAccessBit : 0) + ) + ) + ), + Flags = (Flags0 & ~RowMajorBit) | (LhsFlags & RowMajorBit) + }; + + binary_evaluator(const XprType& xpr) : m_functor(xpr.functor()), m_lhsImpl(xpr.lhs()), m_rhsImpl(xpr.rhs()) @@ -501,14 +485,14 @@ struct evaluator_impl<CwiseBinaryOp<BinaryOp, Lhs, Rhs> > PacketScalar packet(Index row, Index col) const { return m_functor.packetOp(m_lhsImpl.template packet<LoadMode>(row, col), - m_rhsImpl.template packet<LoadMode>(row, col)); + m_rhsImpl.template packet<LoadMode>(row, col)); } template<int LoadMode> PacketScalar packet(Index index) const { return m_functor.packetOp(m_lhsImpl.template packet<LoadMode>(index), - m_rhsImpl.template packet<LoadMode>(index)); + m_rhsImpl.template packet<LoadMode>(index)); } protected: @@ -520,12 +504,18 @@ protected: // -------------------- CwiseUnaryView -------------------- template<typename UnaryOp, typename ArgType> -struct evaluator_impl<CwiseUnaryView<UnaryOp, ArgType> > - : evaluator_impl_base<CwiseUnaryView<UnaryOp, ArgType> > +struct unary_evaluator<CwiseUnaryView<UnaryOp, ArgType>, IndexBased> + : evaluator_base<CwiseUnaryView<UnaryOp, ArgType> > { typedef CwiseUnaryView<UnaryOp, ArgType> XprType; + + enum { + CoeffReadCost = evaluator<ArgType>::CoeffReadCost + functor_traits<UnaryOp>::Cost, + + Flags = (evaluator<ArgType>::Flags & (HereditaryBits | LinearAccessBit | DirectAccessBit)) + }; - evaluator_impl(const XprType& op) + unary_evaluator(const XprType& op) : m_unaryOp(op.functor()), m_argImpl(op.nestedExpression()) { } @@ -561,13 +551,15 @@ protected: // -------------------- Map -------------------- -template<typename Derived, int AccessorsType> -struct evaluator_impl<MapBase<Derived, AccessorsType> > - : evaluator_impl_base<Derived> -{ - typedef MapBase<Derived, AccessorsType> MapType; - typedef Derived XprType; +// FIXME perhaps the PlainObjectType could be provided by Derived::PlainObject ? +// but that might complicate template specialization +template<typename Derived, typename PlainObjectType> +struct mapbase_evaluator; +template<typename Derived, typename PlainObjectType> +struct mapbase_evaluator : evaluator_base<Derived> +{ + typedef Derived XprType; typedef typename XprType::PointerType PointerType; typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; @@ -575,81 +567,121 @@ struct evaluator_impl<MapBase<Derived, AccessorsType> > typedef typename XprType::PacketScalar PacketScalar; typedef typename XprType::PacketReturnType PacketReturnType; - evaluator_impl(const XprType& map) - : m_data(const_cast<PointerType>(map.data())), - m_rowStride(map.rowStride()), - m_colStride(map.colStride()) - { } - enum { - RowsAtCompileTime = XprType::RowsAtCompileTime + IsRowMajor = XprType::RowsAtCompileTime, + ColsAtCompileTime = XprType::ColsAtCompileTime, + CoeffReadCost = NumTraits<Scalar>::ReadCost }; + + mapbase_evaluator(const XprType& map) + : m_data(const_cast<PointerType>(map.data())), + m_xpr(map) + { + EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(evaluator<Derived>::Flags&PacketAccessBit, internal::inner_stride_at_compile_time<Derived>::ret==1), + PACKET_ACCESS_REQUIRES_TO_HAVE_INNER_STRIDE_FIXED_TO_1); + } CoeffReturnType coeff(Index row, Index col) const - { - return m_data[col * m_colStride + row * m_rowStride]; + { + return m_data[col * m_xpr.colStride() + row * m_xpr.rowStride()]; } CoeffReturnType coeff(Index index) const - { - return coeff(RowsAtCompileTime == 1 ? 0 : index, - RowsAtCompileTime == 1 ? index : 0); + { + return m_data[index * m_xpr.innerStride()]; } Scalar& coeffRef(Index row, Index col) - { - return m_data[col * m_colStride + row * m_rowStride]; + { + return m_data[col * m_xpr.colStride() + row * m_xpr.rowStride()]; } Scalar& coeffRef(Index index) - { - return coeffRef(RowsAtCompileTime == 1 ? 0 : index, - RowsAtCompileTime == 1 ? index : 0); + { + return m_data[index * m_xpr.innerStride()]; } template<int LoadMode> PacketReturnType packet(Index row, Index col) const - { - PointerType ptr = m_data + row * m_rowStride + col * m_colStride; + { + PointerType ptr = m_data + row * m_xpr.rowStride() + col * m_xpr.colStride(); return internal::ploadt<PacketScalar, LoadMode>(ptr); } template<int LoadMode> PacketReturnType packet(Index index) const - { - return packet<LoadMode>(RowsAtCompileTime == 1 ? 0 : index, - RowsAtCompileTime == 1 ? index : 0); + { + return internal::ploadt<PacketScalar, LoadMode>(m_data + index * m_xpr.innerStride()); } template<int StoreMode> void writePacket(Index row, Index col, const PacketScalar& x) - { - PointerType ptr = m_data + row * m_rowStride + col * m_colStride; + { + PointerType ptr = m_data + row * m_xpr.rowStride() + col * m_xpr.colStride(); return internal::pstoret<Scalar, PacketScalar, StoreMode>(ptr, x); } template<int StoreMode> void writePacket(Index index, const PacketScalar& x) - { - return writePacket<StoreMode>(RowsAtCompileTime == 1 ? 0 : index, - RowsAtCompileTime == 1 ? index : 0, - x); + { + internal::pstoret<Scalar, PacketScalar, StoreMode>(m_data + index * m_xpr.innerStride(), x); } protected: PointerType m_data; - int m_rowStride; - int m_colStride; + const XprType& m_xpr; }; template<typename PlainObjectType, int MapOptions, typename StrideType> -struct evaluator_impl<Map<PlainObjectType, MapOptions, StrideType> > - : public evaluator_impl<MapBase<Map<PlainObjectType, MapOptions, StrideType> > > +struct evaluator<Map<PlainObjectType, MapOptions, StrideType> > + : public mapbase_evaluator<Map<PlainObjectType, MapOptions, StrideType>, PlainObjectType> { typedef Map<PlainObjectType, MapOptions, StrideType> XprType; + typedef typename XprType::Scalar Scalar; + + enum { + InnerStrideAtCompileTime = StrideType::InnerStrideAtCompileTime == 0 + ? int(PlainObjectType::InnerStrideAtCompileTime) + : int(StrideType::InnerStrideAtCompileTime), + OuterStrideAtCompileTime = StrideType::OuterStrideAtCompileTime == 0 + ? int(PlainObjectType::OuterStrideAtCompileTime) + : int(StrideType::OuterStrideAtCompileTime), + HasNoInnerStride = InnerStrideAtCompileTime == 1, + HasNoOuterStride = StrideType::OuterStrideAtCompileTime == 0, + HasNoStride = HasNoInnerStride && HasNoOuterStride, + IsAligned = bool(EIGEN_ALIGN) && ((int(MapOptions)&Aligned)==Aligned), + IsDynamicSize = PlainObjectType::SizeAtCompileTime==Dynamic, + KeepsPacketAccess = bool(HasNoInnerStride) + && ( bool(IsDynamicSize) + || HasNoOuterStride + || ( OuterStrideAtCompileTime!=Dynamic + && ((static_cast<int>(sizeof(Scalar))*OuterStrideAtCompileTime)%EIGEN_ALIGN_BYTES)==0 ) ), + Flags0 = evaluator<PlainObjectType>::Flags, + Flags1 = IsAligned ? (int(Flags0) | AlignedBit) : (int(Flags0) & ~AlignedBit), + Flags2 = (bool(HasNoStride) || bool(PlainObjectType::IsVectorAtCompileTime)) + ? int(Flags1) : int(Flags1 & ~LinearAccessBit), + Flags = KeepsPacketAccess ? int(Flags2) : (int(Flags2) & ~PacketAccessBit) + }; + + evaluator(const XprType& map) + : mapbase_evaluator<XprType, PlainObjectType>(map) + { } +}; + +// -------------------- Ref -------------------- + +template<typename PlainObjectType, int RefOptions, typename StrideType> +struct evaluator<Ref<PlainObjectType, RefOptions, StrideType> > + : public mapbase_evaluator<Ref<PlainObjectType, RefOptions, StrideType>, PlainObjectType> +{ + typedef Ref<PlainObjectType, RefOptions, StrideType> XprType; + + enum { + Flags = evaluator<Map<PlainObjectType, RefOptions, StrideType> >::Flags + }; - evaluator_impl(const XprType& map) - : evaluator_impl<MapBase<XprType> >(map) + evaluator(const XprType& ref) + : mapbase_evaluator<XprType, PlainObjectType>(ref) { } }; @@ -659,21 +691,68 @@ template<typename ArgType, int BlockRows, int BlockCols, bool InnerPanel, bool HasDirectAccess = internal::has_direct_access<ArgType>::ret> struct block_evaluator; template<typename ArgType, int BlockRows, int BlockCols, bool InnerPanel> -struct evaluator_impl<Block<ArgType, BlockRows, BlockCols, InnerPanel> > +struct evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel> > : block_evaluator<ArgType, BlockRows, BlockCols, InnerPanel> { typedef Block<ArgType, BlockRows, BlockCols, InnerPanel> XprType; + typedef typename XprType::Scalar Scalar; + + enum { + CoeffReadCost = evaluator<ArgType>::CoeffReadCost, + + RowsAtCompileTime = traits<XprType>::RowsAtCompileTime, + ColsAtCompileTime = traits<XprType>::ColsAtCompileTime, + MaxRowsAtCompileTime = traits<XprType>::MaxRowsAtCompileTime, + MaxColsAtCompileTime = traits<XprType>::MaxColsAtCompileTime, + + ArgTypeIsRowMajor = (int(evaluator<ArgType>::Flags)&RowMajorBit) != 0, + IsRowMajor = (MaxRowsAtCompileTime==1 && MaxColsAtCompileTime!=1) ? 1 + : (MaxColsAtCompileTime==1 && MaxRowsAtCompileTime!=1) ? 0 + : ArgTypeIsRowMajor, + HasSameStorageOrderAsArgType = (IsRowMajor == ArgTypeIsRowMajor), + InnerSize = IsRowMajor ? int(ColsAtCompileTime) : int(RowsAtCompileTime), + InnerStrideAtCompileTime = HasSameStorageOrderAsArgType + ? int(inner_stride_at_compile_time<ArgType>::ret) + : int(outer_stride_at_compile_time<ArgType>::ret), + OuterStrideAtCompileTime = HasSameStorageOrderAsArgType + ? int(outer_stride_at_compile_time<ArgType>::ret) + : int(inner_stride_at_compile_time<ArgType>::ret), + MaskPacketAccessBit = (InnerSize == Dynamic || (InnerSize % packet_traits<Scalar>::size) == 0) + && (InnerStrideAtCompileTime == 1) + ? PacketAccessBit : 0, + + MaskAlignedBit = (InnerPanel && (OuterStrideAtCompileTime!=Dynamic) && (((OuterStrideAtCompileTime * int(sizeof(Scalar))) % EIGEN_ALIGN_BYTES) == 0)) ? AlignedBit : 0, + FlagsLinearAccessBit = (RowsAtCompileTime == 1 || ColsAtCompileTime == 1 || (InnerPanel && (traits<XprType>::Flags&LinearAccessBit))) ? LinearAccessBit : 0, + FlagsRowMajorBit = XprType::Flags&RowMajorBit, + Flags0 = evaluator<ArgType>::Flags & ( (HereditaryBits & ~RowMajorBit) | + DirectAccessBit | + MaskPacketAccessBit | + MaskAlignedBit), + Flags = Flags0 | FlagsLinearAccessBit | FlagsRowMajorBit + }; typedef block_evaluator<ArgType, BlockRows, BlockCols, InnerPanel> block_evaluator_type; - evaluator_impl(const XprType& block) : block_evaluator_type(block) {} + evaluator(const XprType& block) : block_evaluator_type(block) {} }; +// no direct-access => dispatch to a unary evaluator template<typename ArgType, int BlockRows, int BlockCols, bool InnerPanel> struct block_evaluator<ArgType, BlockRows, BlockCols, InnerPanel, /*HasDirectAccess*/ false> - : evaluator_impl_base<Block<ArgType, BlockRows, BlockCols, InnerPanel> > + : unary_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel> > { typedef Block<ArgType, BlockRows, BlockCols, InnerPanel> XprType; block_evaluator(const XprType& block) + : unary_evaluator<XprType>(block) + {} +}; + +template<typename ArgType, int BlockRows, int BlockCols, bool InnerPanel> +struct unary_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>, IndexBased> + : evaluator_base<Block<ArgType, BlockRows, BlockCols, InnerPanel> > +{ + typedef Block<ArgType, BlockRows, BlockCols, InnerPanel> XprType; + + unary_evaluator(const XprType& block) : m_argImpl(block.nestedExpression()), m_startRow(block.startRow()), m_startCol(block.startCol()) @@ -696,8 +775,7 @@ struct block_evaluator<ArgType, BlockRows, BlockCols, InnerPanel, /*HasDirectAcc CoeffReturnType coeff(Index index) const { - return coeff(RowsAtCompileTime == 1 ? 0 : index, - RowsAtCompileTime == 1 ? index : 0); + return coeff(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0); } Scalar& coeffRef(Index row, Index col) @@ -707,8 +785,7 @@ struct block_evaluator<ArgType, BlockRows, BlockCols, InnerPanel, /*HasDirectAcc Scalar& coeffRef(Index index) { - return coeffRef(RowsAtCompileTime == 1 ? 0 : index, - RowsAtCompileTime == 1 ? index : 0); + return coeffRef(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0); } template<int LoadMode> @@ -721,7 +798,7 @@ struct block_evaluator<ArgType, BlockRows, BlockCols, InnerPanel, /*HasDirectAcc PacketReturnType packet(Index index) const { return packet<LoadMode>(RowsAtCompileTime == 1 ? 0 : index, - RowsAtCompileTime == 1 ? index : 0); + RowsAtCompileTime == 1 ? index : 0); } template<int StoreMode> @@ -734,8 +811,8 @@ struct block_evaluator<ArgType, BlockRows, BlockCols, InnerPanel, /*HasDirectAcc void writePacket(Index index, const PacketScalar& x) { return writePacket<StoreMode>(RowsAtCompileTime == 1 ? 0 : index, - RowsAtCompileTime == 1 ? index : 0, - x); + RowsAtCompileTime == 1 ? index : 0, + x); } protected: @@ -749,24 +826,38 @@ protected: template<typename ArgType, int BlockRows, int BlockCols, bool InnerPanel> struct block_evaluator<ArgType, BlockRows, BlockCols, InnerPanel, /* HasDirectAccess */ true> - : evaluator_impl<MapBase<Block<ArgType, BlockRows, BlockCols, InnerPanel> > > + : mapbase_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>, + typename Block<ArgType, BlockRows, BlockCols, InnerPanel>::PlainObject> { typedef Block<ArgType, BlockRows, BlockCols, InnerPanel> XprType; block_evaluator(const XprType& block) - : evaluator_impl<MapBase<XprType> >(block) - { } + : mapbase_evaluator<XprType, typename XprType::PlainObject>(block) + { + // FIXME this should be an internal assertion + eigen_assert(EIGEN_IMPLIES(evaluator<XprType>::Flags&AlignedBit, (size_t(block.data()) % EIGEN_ALIGN_BYTES) == 0) && "data is not aligned"); + } }; // -------------------- Select -------------------- +// TODO shall we introduce a ternary_evaluator? +// TODO enable vectorization for Select template<typename ConditionMatrixType, typename ThenMatrixType, typename ElseMatrixType> -struct evaluator_impl<Select<ConditionMatrixType, ThenMatrixType, ElseMatrixType> > +struct evaluator<Select<ConditionMatrixType, ThenMatrixType, ElseMatrixType> > + : evaluator_base<Select<ConditionMatrixType, ThenMatrixType, ElseMatrixType> > { typedef Select<ConditionMatrixType, ThenMatrixType, ElseMatrixType> XprType; + enum { + CoeffReadCost = evaluator<ConditionMatrixType>::CoeffReadCost + + EIGEN_SIZE_MAX(evaluator<ThenMatrixType>::CoeffReadCost, + evaluator<ElseMatrixType>::CoeffReadCost), + + Flags = (unsigned int)evaluator<ThenMatrixType>::Flags & evaluator<ElseMatrixType>::Flags & HereditaryBits + }; - evaluator_impl(const XprType& select) + evaluator(const XprType& select) : m_conditionImpl(select.conditionMatrix()), m_thenImpl(select.thenMatrix()), m_elseImpl(select.elseMatrix()) @@ -801,20 +892,32 @@ protected: // -------------------- Replicate -------------------- template<typename ArgType, int RowFactor, int ColFactor> -struct evaluator_impl<Replicate<ArgType, RowFactor, ColFactor> > +struct unary_evaluator<Replicate<ArgType, RowFactor, ColFactor> > + : evaluator_base<Replicate<ArgType, RowFactor, ColFactor> > { typedef Replicate<ArgType, RowFactor, ColFactor> XprType; - - evaluator_impl(const XprType& replicate) - : m_argImpl(replicate.nestedExpression()), - m_rows(replicate.nestedExpression().rows()), - m_cols(replicate.nestedExpression().cols()) - { } - typedef typename XprType::Index Index; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename XprType::PacketReturnType PacketReturnType; + enum { + Factor = (RowFactor==Dynamic || ColFactor==Dynamic) ? Dynamic : RowFactor*ColFactor + }; + typedef typename internal::nested_eval<ArgType,Factor>::type ArgTypeNested; + typedef typename internal::remove_all<ArgTypeNested>::type ArgTypeNestedCleaned; + + enum { + CoeffReadCost = evaluator<ArgTypeNestedCleaned>::CoeffReadCost, + + Flags = (evaluator<ArgTypeNestedCleaned>::Flags & HereditaryBits & ~RowMajorBit) | (traits<XprType>::Flags & RowMajorBit) + }; + unary_evaluator(const XprType& replicate) + : m_arg(replicate.nestedExpression()), + m_argImpl(m_arg), + m_rows(replicate.nestedExpression().rows()), + m_cols(replicate.nestedExpression().cols()) + {} + CoeffReturnType coeff(Index row, Index col) const { // try to avoid using modulo; this is a pure optimization strategy @@ -842,9 +945,10 @@ struct evaluator_impl<Replicate<ArgType, RowFactor, ColFactor> > } protected: - typename evaluator<ArgType>::nestedType m_argImpl; - const variable_if_dynamic<Index, XprType::RowsAtCompileTime> m_rows; - const variable_if_dynamic<Index, XprType::ColsAtCompileTime> m_cols; + const ArgTypeNested m_arg; // FIXME is it OK to store both the argument and its evaluator?? (we have the same situation in evaluator_product) + typename evaluator<ArgTypeNestedCleaned>::nestedType m_argImpl; + const variable_if_dynamic<Index, ArgType::RowsAtCompileTime> m_rows; + const variable_if_dynamic<Index, ArgType::ColsAtCompileTime> m_cols; }; @@ -855,13 +959,25 @@ protected: // the row() and col() member functions. template< typename ArgType, typename MemberOp, int Direction> -struct evaluator_impl<PartialReduxExpr<ArgType, MemberOp, Direction> > +struct evaluator<PartialReduxExpr<ArgType, MemberOp, Direction> > + : evaluator_base<PartialReduxExpr<ArgType, MemberOp, Direction> > { typedef PartialReduxExpr<ArgType, MemberOp, Direction> XprType; + typedef typename XprType::Scalar InputScalar; + enum { + TraversalSize = Direction==int(Vertical) ? int(ArgType::RowsAtCompileTime) : int(XprType::ColsAtCompileTime) + }; + typedef typename MemberOp::template Cost<InputScalar,int(TraversalSize)> CostOpType; + enum { + CoeffReadCost = TraversalSize==Dynamic ? Dynamic + : TraversalSize * evaluator<ArgType>::CoeffReadCost + int(CostOpType::value), + + Flags = (traits<XprType>::Flags&RowMajorBit) | (evaluator<ArgType>::Flags&HereditaryBits) + }; - evaluator_impl(const XprType expr) + evaluator(const XprType expr) : m_expr(expr) - { } + {} typedef typename XprType::Index Index; typedef typename XprType::CoeffReturnType CoeffReturnType; @@ -883,16 +999,20 @@ protected: // -------------------- MatrixWrapper and ArrayWrapper -------------------- // -// evaluator_impl_wrapper_base<T> is a common base class for the +// evaluator_wrapper_base<T> is a common base class for the // MatrixWrapper and ArrayWrapper evaluators. template<typename XprType> -struct evaluator_impl_wrapper_base - : evaluator_impl_base<XprType> +struct evaluator_wrapper_base + : evaluator_base<XprType> { typedef typename remove_all<typename XprType::NestedExpressionType>::type ArgType; + enum { + CoeffReadCost = evaluator<ArgType>::CoeffReadCost, + Flags = evaluator<ArgType>::Flags + }; - evaluator_impl_wrapper_base(const ArgType& arg) : m_argImpl(arg) {} + evaluator_wrapper_base(const ArgType& arg) : m_argImpl(arg) {} typedef typename ArgType::Index Index; typedef typename ArgType::Scalar Scalar; @@ -949,24 +1069,24 @@ protected: }; template<typename TArgType> -struct evaluator_impl<MatrixWrapper<TArgType> > - : evaluator_impl_wrapper_base<MatrixWrapper<TArgType> > +struct unary_evaluator<MatrixWrapper<TArgType> > + : evaluator_wrapper_base<MatrixWrapper<TArgType> > { typedef MatrixWrapper<TArgType> XprType; - evaluator_impl(const XprType& wrapper) - : evaluator_impl_wrapper_base<MatrixWrapper<TArgType> >(wrapper.nestedExpression()) + unary_evaluator(const XprType& wrapper) + : evaluator_wrapper_base<MatrixWrapper<TArgType> >(wrapper.nestedExpression()) { } }; template<typename TArgType> -struct evaluator_impl<ArrayWrapper<TArgType> > - : evaluator_impl_wrapper_base<ArrayWrapper<TArgType> > +struct unary_evaluator<ArrayWrapper<TArgType> > + : evaluator_wrapper_base<ArrayWrapper<TArgType> > { typedef ArrayWrapper<TArgType> XprType; - evaluator_impl(const XprType& wrapper) - : evaluator_impl_wrapper_base<ArrayWrapper<TArgType> >(wrapper.nestedExpression()) + unary_evaluator(const XprType& wrapper) + : evaluator_wrapper_base<ArrayWrapper<TArgType> >(wrapper.nestedExpression()) { } }; @@ -977,8 +1097,8 @@ struct evaluator_impl<ArrayWrapper<TArgType> > template<typename PacketScalar, bool ReversePacket> struct reverse_packet_cond; template<typename ArgType, int Direction> -struct evaluator_impl<Reverse<ArgType, Direction> > - : evaluator_impl_base<Reverse<ArgType, Direction> > +struct unary_evaluator<Reverse<ArgType, Direction> > + : evaluator_base<Reverse<ArgType, Direction> > { typedef Reverse<ArgType, Direction> XprType; typedef typename XprType::Index Index; @@ -997,11 +1117,21 @@ struct evaluator_impl<Reverse<ArgType, Direction> > OffsetCol = ReverseCol && IsRowMajor ? PacketSize : 1, ReversePacket = (Direction == BothDirections) || ((Direction == Vertical) && IsColMajor) - || ((Direction == Horizontal) && IsRowMajor) + || ((Direction == Horizontal) && IsRowMajor), + + CoeffReadCost = evaluator<ArgType>::CoeffReadCost, + + // let's enable LinearAccess only with vectorization because of the product overhead + // FIXME enable DirectAccess with negative strides? + Flags0 = evaluator<ArgType>::Flags, + LinearAccess = ( (Direction==BothDirections) && (int(Flags0)&PacketAccessBit) ) + ? LinearAccessBit : 0, + + Flags = int(Flags0) & (HereditaryBits | PacketAccessBit | LinearAccess) }; typedef internal::reverse_packet_cond<PacketScalar,ReversePacket> reverse_packet; - evaluator_impl(const XprType& reverse) + unary_evaluator(const XprType& reverse) : m_argImpl(reverse.nestedExpression()), m_rows(ReverseRow ? reverse.nestedExpression().rows() : 0), m_cols(ReverseCol ? reverse.nestedExpression().cols() : 0) @@ -1010,7 +1140,7 @@ struct evaluator_impl<Reverse<ArgType, Direction> > CoeffReturnType coeff(Index row, Index col) const { return m_argImpl.coeff(ReverseRow ? m_rows.value() - row - 1 : row, - ReverseCol ? m_cols.value() - col - 1 : col); + ReverseCol ? m_cols.value() - col - 1 : col); } CoeffReturnType coeff(Index index) const @@ -1021,7 +1151,7 @@ struct evaluator_impl<Reverse<ArgType, Direction> > Scalar& coeffRef(Index row, Index col) { return m_argImpl.coeffRef(ReverseRow ? m_rows.value() - row - 1 : row, - ReverseCol ? m_cols.value() - col - 1 : col); + ReverseCol ? m_cols.value() - col - 1 : col); } Scalar& coeffRef(Index index) @@ -1071,12 +1201,18 @@ protected: // -------------------- Diagonal -------------------- template<typename ArgType, int DiagIndex> -struct evaluator_impl<Diagonal<ArgType, DiagIndex> > - : evaluator_impl_base<Diagonal<ArgType, DiagIndex> > +struct evaluator<Diagonal<ArgType, DiagIndex> > + : evaluator_base<Diagonal<ArgType, DiagIndex> > { typedef Diagonal<ArgType, DiagIndex> XprType; + + enum { + CoeffReadCost = evaluator<ArgType>::CoeffReadCost, + + Flags = (unsigned int)evaluator<ArgType>::Flags & (HereditaryBits | LinearAccessBit | DirectAccessBit) & ~RowMajorBit + }; - evaluator_impl(const XprType& diagonal) + evaluator(const XprType& diagonal) : m_argImpl(diagonal.nestedExpression()), m_index(diagonal.index()) { } @@ -1114,6 +1250,86 @@ private: EIGEN_STRONG_INLINE Index colOffset() const { return m_index.value() > 0 ? m_index.value() : 0; } }; + +//---------------------------------------------------------------------- +// deprecated code +//---------------------------------------------------------------------- + +// -------------------- EvalToTemp -------------------- + +// expression class for evaluating nested expression to a temporary + +template<typename ArgType> class EvalToTemp; + +template<typename ArgType> +struct traits<EvalToTemp<ArgType> > + : public traits<ArgType> +{ }; + +template<typename ArgType> +class EvalToTemp + : public dense_xpr_base<EvalToTemp<ArgType> >::type +{ + public: + + typedef typename dense_xpr_base<EvalToTemp>::type Base; + EIGEN_GENERIC_PUBLIC_INTERFACE(EvalToTemp) + + EvalToTemp(const ArgType& arg) + : m_arg(arg) + { } + + const ArgType& arg() const + { + return m_arg; + } + + Index rows() const + { + return m_arg.rows(); + } + + Index cols() const + { + return m_arg.cols(); + } + + private: + const ArgType& m_arg; +}; + +template<typename ArgType> +struct evaluator<EvalToTemp<ArgType> > + : public evaluator<typename ArgType::PlainObject>::type +{ + typedef EvalToTemp<ArgType> XprType; + typedef typename ArgType::PlainObject PlainObject; + typedef typename evaluator<PlainObject>::type Base; + + typedef evaluator type; + typedef evaluator nestedType; + + evaluator(const XprType& xpr) + : m_result(xpr.rows(), xpr.cols()) + { + ::new (static_cast<Base*>(this)) Base(m_result); + // TODO we should simply do m_result(xpr.arg()); + call_dense_assignment_loop(m_result, xpr.arg()); + } + + // This constructor is used when nesting an EvalTo evaluator in another evaluator + evaluator(const ArgType& arg) + : m_result(arg.rows(), arg.cols()) + { + ::new (static_cast<Base*>(this)) Base(m_result); + // TODO we should simply do m_result(xpr.arg()); + call_dense_assignment_loop(m_result, arg); + } + +protected: + PlainObject m_result; +}; + } // namespace internal } // end namespace Eigen diff --git a/Eigen/src/Core/CwiseBinaryOp.h b/Eigen/src/Core/CwiseBinaryOp.h index e20daacc8..b4662907e 100644 --- a/Eigen/src/Core/CwiseBinaryOp.h +++ b/Eigen/src/Core/CwiseBinaryOp.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com> // // This Source Code Form is subject to the terms of the Mozilla @@ -56,8 +56,9 @@ struct traits<CwiseBinaryOp<BinaryOp, Lhs, Rhs> > typename Rhs::Scalar ) >::type Scalar; - typedef typename promote_storage_type<typename traits<Lhs>::StorageKind, - typename traits<Rhs>::StorageKind>::ret StorageKind; + typedef typename cwise_promote_storage_type<typename traits<Lhs>::StorageKind, + typename traits<Rhs>::StorageKind, + BinaryOp>::ret StorageKind; typedef typename promote_index_type<typename traits<Lhs>::Index, typename traits<Rhs>::Index>::type Index; typedef typename Lhs::Nested LhsNested; @@ -65,8 +66,7 @@ struct traits<CwiseBinaryOp<BinaryOp, Lhs, Rhs> > typedef typename remove_reference<LhsNested>::type _LhsNested; typedef typename remove_reference<RhsNested>::type _RhsNested; enum { - LhsCoeffReadCost = _LhsNested::CoeffReadCost, - RhsCoeffReadCost = _RhsNested::CoeffReadCost, +#ifndef EIGEN_TEST_EVALUATORS LhsFlags = _LhsNested::Flags, RhsFlags = _RhsNested::Flags, SameType = is_same<typename _LhsNested::Scalar,typename _RhsNested::Scalar>::value, @@ -81,44 +81,42 @@ struct traits<CwiseBinaryOp<BinaryOp, Lhs, Rhs> > ) ), Flags = (Flags0 & ~RowMajorBit) | (LhsFlags & RowMajorBit), + + LhsCoeffReadCost = _LhsNested::CoeffReadCost, + RhsCoeffReadCost = _RhsNested::CoeffReadCost, CoeffReadCost = LhsCoeffReadCost + RhsCoeffReadCost + functor_traits<BinaryOp>::Cost +#else + Flags = _LhsNested::Flags & RowMajorBit +#endif }; }; } // end namespace internal -// we require Lhs and Rhs to have the same scalar type. Currently there is no example of a binary functor -// that would take two operands of different types. If there were such an example, then this check should be -// moved to the BinaryOp functors, on a per-case basis. This would however require a change in the BinaryOp functors, as -// currently they take only one typename Scalar template parameter. -// It is tempting to always allow mixing different types but remember that this is often impossible in the vectorized paths. -// So allowing mixing different types gives very unexpected errors when enabling vectorization, when the user tries to -// add together a float matrix and a double matrix. -#define EIGEN_CHECK_BINARY_COMPATIBILIY(BINOP,LHS,RHS) \ - EIGEN_STATIC_ASSERT((internal::functor_is_product_like<BINOP>::ret \ - ? int(internal::scalar_product_traits<LHS, RHS>::Defined) \ - : int(internal::is_same<LHS, RHS>::value)), \ - YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) - template<typename BinaryOp, typename Lhs, typename Rhs, typename StorageKind> class CwiseBinaryOpImpl; -template<typename BinaryOp, typename Lhs, typename Rhs> +template<typename BinaryOp, typename LhsType, typename RhsType> class CwiseBinaryOp : internal::no_assignment_operator, public CwiseBinaryOpImpl< - BinaryOp, Lhs, Rhs, - typename internal::promote_storage_type<typename internal::traits<Lhs>::StorageKind, - typename internal::traits<Rhs>::StorageKind>::ret> + BinaryOp, LhsType, RhsType, + typename internal::cwise_promote_storage_type<typename internal::traits<LhsType>::StorageKind, + typename internal::traits<RhsType>::StorageKind, + BinaryOp>::ret> { public: + + typedef typename internal::remove_all<LhsType>::type Lhs; + typedef typename internal::remove_all<RhsType>::type Rhs; typedef typename CwiseBinaryOpImpl< - BinaryOp, Lhs, Rhs, - typename internal::promote_storage_type<typename internal::traits<Lhs>::StorageKind, - typename internal::traits<Rhs>::StorageKind>::ret>::Base Base; + BinaryOp, LhsType, RhsType, + typename internal::cwise_promote_storage_type<typename internal::traits<LhsType>::StorageKind, + typename internal::traits<Rhs>::StorageKind, + BinaryOp>::ret>::Base Base; EIGEN_GENERIC_PUBLIC_INTERFACE(CwiseBinaryOp) - typedef typename internal::nested<Lhs>::type LhsNested; - typedef typename internal::nested<Rhs>::type RhsNested; + typedef typename internal::nested<LhsType>::type LhsNested; + typedef typename internal::nested<RhsType>::type RhsNested; typedef typename internal::remove_reference<LhsNested>::type _LhsNested; typedef typename internal::remove_reference<RhsNested>::type _RhsNested; @@ -165,6 +163,7 @@ class CwiseBinaryOp : internal::no_assignment_operator, const BinaryOp m_functor; }; +#ifndef EIGEN_TEST_EVALUATORS template<typename BinaryOp, typename Lhs, typename Rhs> class CwiseBinaryOpImpl<BinaryOp, Lhs, Rhs, Dense> : public internal::dense_xpr_base<CwiseBinaryOp<BinaryOp, Lhs, Rhs> >::type @@ -203,6 +202,16 @@ class CwiseBinaryOpImpl<BinaryOp, Lhs, Rhs, Dense> derived().rhs().template packet<LoadMode>(index)); } }; +#else +// Generic API dispatcher +template<typename BinaryOp, typename Lhs, typename Rhs, typename StorageKind> +class CwiseBinaryOpImpl + : public internal::generic_xpr_base<CwiseBinaryOp<BinaryOp, Lhs, Rhs> >::type +{ +public: + typedef typename internal::generic_xpr_base<CwiseBinaryOp<BinaryOp, Lhs, Rhs> >::type Base; +}; +#endif /** replaces \c *this by \c *this - \a other. * @@ -213,8 +222,12 @@ template<typename OtherDerived> EIGEN_STRONG_INLINE Derived & MatrixBase<Derived>::operator-=(const MatrixBase<OtherDerived> &other) { +#ifdef EIGEN_TEST_EVALUATORS + call_assignment(derived(), other.derived(), internal::sub_assign_op<Scalar>()); +#else SelfCwiseBinaryOp<internal::scalar_difference_op<Scalar>, Derived, OtherDerived> tmp(derived()); tmp = other.derived(); +#endif return derived(); } @@ -227,8 +240,12 @@ template<typename OtherDerived> EIGEN_STRONG_INLINE Derived & MatrixBase<Derived>::operator+=(const MatrixBase<OtherDerived>& other) { +#ifdef EIGEN_TEST_EVALUATORS + call_assignment(derived(), other.derived(), internal::add_assign_op<Scalar>()); +#else SelfCwiseBinaryOp<internal::scalar_sum_op<Scalar>, Derived, OtherDerived> tmp(derived()); tmp = other.derived(); +#endif return derived(); } diff --git a/Eigen/src/Core/CwiseNullaryOp.h b/Eigen/src/Core/CwiseNullaryOp.h index 124383114..f9f127cc2 100644 --- a/Eigen/src/Core/CwiseNullaryOp.h +++ b/Eigen/src/Core/CwiseNullaryOp.h @@ -35,12 +35,16 @@ template<typename NullaryOp, typename PlainObjectType> struct traits<CwiseNullaryOp<NullaryOp, PlainObjectType> > : traits<PlainObjectType> { enum { +#ifndef EIGEN_TEST_EVALUATORS Flags = (traits<PlainObjectType>::Flags & ( HereditaryBits | (functor_has_linear_access<NullaryOp>::ret ? LinearAccessBit : 0) | (functor_traits<NullaryOp>::PacketAccess ? PacketAccessBit : 0))) | (functor_traits<NullaryOp>::IsRepeatable ? 0 : EvalBeforeNestingBit), CoeffReadCost = functor_traits<NullaryOp>::Cost +#else + Flags = traits<PlainObjectType>::Flags & RowMajorBit +#endif }; }; } diff --git a/Eigen/src/Core/CwiseUnaryOp.h b/Eigen/src/Core/CwiseUnaryOp.h index aa7df197f..c2bc47c93 100644 --- a/Eigen/src/Core/CwiseUnaryOp.h +++ b/Eigen/src/Core/CwiseUnaryOp.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2010 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com> // // This Source Code Form is subject to the terms of the Mozilla @@ -44,10 +44,14 @@ struct traits<CwiseUnaryOp<UnaryOp, XprType> > typedef typename XprType::Nested XprTypeNested; typedef typename remove_reference<XprTypeNested>::type _XprTypeNested; enum { +#ifndef EIGEN_TEST_EVALUATORS Flags = _XprTypeNested::Flags & ( HereditaryBits | LinearAccessBit | AlignedBit | (functor_traits<UnaryOp>::PacketAccess ? PacketAccessBit : 0)), CoeffReadCost = _XprTypeNested::CoeffReadCost + functor_traits<UnaryOp>::Cost +#else + Flags = _XprTypeNested::Flags & RowMajorBit +#endif }; }; } @@ -63,6 +67,7 @@ class CwiseUnaryOp : internal::no_assignment_operator, typedef typename CwiseUnaryOpImpl<UnaryOp, XprType,typename internal::traits<XprType>::StorageKind>::Base Base; EIGEN_GENERIC_PUBLIC_INTERFACE(CwiseUnaryOp) + typedef typename internal::remove_all<XprType>::type NestedExpression; EIGEN_DEVICE_FUNC inline CwiseUnaryOp(const XprType& xpr, const UnaryOp& func = UnaryOp()) @@ -92,6 +97,7 @@ class CwiseUnaryOp : internal::no_assignment_operator, const UnaryOp m_functor; }; +#ifndef EIGEN_TEST_EVALUATORS // This is the generic implementation for dense storage. // It can be used for any expression types implementing the dense concept. template<typename UnaryOp, typename XprType> @@ -103,7 +109,7 @@ class CwiseUnaryOpImpl<UnaryOp,XprType,Dense> typedef CwiseUnaryOp<UnaryOp, XprType> Derived; typedef typename internal::dense_xpr_base<CwiseUnaryOp<UnaryOp, XprType> >::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(Derived) - + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar coeff(Index rowId, Index colId) const { @@ -129,6 +135,16 @@ class CwiseUnaryOpImpl<UnaryOp,XprType,Dense> return derived().functor().packetOp(derived().nestedExpression().template packet<LoadMode>(index)); } }; +#else // EIGEN_TEST_EVALUATORS +// Generic API dispatcher +template<typename UnaryOp, typename XprType, typename StorageKind> +class CwiseUnaryOpImpl + : public internal::generic_xpr_base<CwiseUnaryOp<UnaryOp, XprType> >::type +{ +public: + typedef typename internal::generic_xpr_base<CwiseUnaryOp<UnaryOp, XprType> >::type Base; +}; +#endif // EIGEN_TEST_EVALUATORS } // end namespace Eigen diff --git a/Eigen/src/Core/CwiseUnaryView.h b/Eigen/src/Core/CwiseUnaryView.h index b2638d326..99cc03ac1 100644 --- a/Eigen/src/Core/CwiseUnaryView.h +++ b/Eigen/src/Core/CwiseUnaryView.h @@ -37,8 +37,12 @@ struct traits<CwiseUnaryView<ViewOp, MatrixType> > typedef typename MatrixType::Nested MatrixTypeNested; typedef typename remove_all<MatrixTypeNested>::type _MatrixTypeNested; enum { +#ifndef EIGEN_TEST_EVALUATORS Flags = (traits<_MatrixTypeNested>::Flags & (HereditaryBits | LvalueBit | LinearAccessBit | DirectAccessBit)), CoeffReadCost = traits<_MatrixTypeNested>::CoeffReadCost + functor_traits<ViewOp>::Cost, +#else + Flags = traits<_MatrixTypeNested>::Flags & (RowMajorBit | LvalueBit | DirectAccessBit), // FIXME DirectAccessBit should not be handled by expressions +#endif MatrixTypeInnerStride = inner_stride_at_compile_time<MatrixType>::ret, // need to cast the sizeof's from size_t to int explicitly, otherwise: // "error: no integral type can represent all of the enumerator values @@ -62,6 +66,7 @@ class CwiseUnaryView : public CwiseUnaryViewImpl<ViewOp, MatrixType, typename in typedef typename CwiseUnaryViewImpl<ViewOp, MatrixType,typename internal::traits<MatrixType>::StorageKind>::Base Base; EIGEN_GENERIC_PUBLIC_INTERFACE(CwiseUnaryView) + typedef typename internal::remove_all<MatrixType>::type NestedExpression; inline CwiseUnaryView(const MatrixType& mat, const ViewOp& func = ViewOp()) : m_matrix(mat), m_functor(func) {} @@ -88,6 +93,17 @@ class CwiseUnaryView : public CwiseUnaryViewImpl<ViewOp, MatrixType, typename in ViewOp m_functor; }; +#ifdef EIGEN_TEST_EVALUATORS +// Generic API dispatcher +template<typename ViewOp, typename XprType, typename StorageKind> +class CwiseUnaryViewImpl + : public internal::generic_xpr_base<CwiseUnaryView<ViewOp, XprType> >::type +{ +public: + typedef typename internal::generic_xpr_base<CwiseUnaryView<ViewOp, XprType> >::type Base; +}; +#endif + template<typename ViewOp, typename MatrixType> class CwiseUnaryViewImpl<ViewOp,MatrixType,Dense> : public internal::dense_xpr_base< CwiseUnaryView<ViewOp, MatrixType> >::type @@ -100,8 +116,8 @@ class CwiseUnaryViewImpl<ViewOp,MatrixType,Dense> EIGEN_DENSE_PUBLIC_INTERFACE(Derived) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(CwiseUnaryViewImpl) - inline Scalar* data() { return &coeffRef(0); } - inline const Scalar* data() const { return &coeff(0); } + inline Scalar* data() { return &(this->coeffRef(0)); } + inline const Scalar* data() const { return &(this->coeff(0)); } inline Index innerStride() const { @@ -112,6 +128,8 @@ class CwiseUnaryViewImpl<ViewOp,MatrixType,Dense> { return derived().nestedExpression().outerStride() * sizeof(typename internal::traits<MatrixType>::Scalar) / sizeof(Scalar); } + +#ifndef EIGEN_TEST_EVALUATORS EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { @@ -132,6 +150,8 @@ class CwiseUnaryViewImpl<ViewOp,MatrixType,Dense> { return derived().functor()(const_cast_derived().nestedExpression().coeffRef(index)); } + +#endif }; } // end namespace Eigen diff --git a/Eigen/src/Core/DenseBase.h b/Eigen/src/Core/DenseBase.h index bd5dd14ed..14643b5a8 100644 --- a/Eigen/src/Core/DenseBase.h +++ b/Eigen/src/Core/DenseBase.h @@ -74,6 +74,7 @@ template<typename Derived> class DenseBase using Base::colIndexByOuterInner; using Base::coeff; using Base::coeffByOuterInner; +#ifndef EIGEN_TEST_EVALUATORS using Base::packet; using Base::packetByOuterInner; using Base::writePacket; @@ -84,6 +85,7 @@ template<typename Derived> class DenseBase using Base::copyCoeffByOuterInner; using Base::copyPacket; using Base::copyPacketByOuterInner; +#endif using Base::operator(); using Base::operator[]; using Base::x; @@ -169,10 +171,12 @@ template<typename Derived> class DenseBase InnerSizeAtCompileTime = int(IsVectorAtCompileTime) ? int(SizeAtCompileTime) : int(IsRowMajor) ? int(ColsAtCompileTime) : int(RowsAtCompileTime), +#ifndef EIGEN_TEST_EVALUATORS CoeffReadCost = internal::traits<Derived>::CoeffReadCost, /**< This is a rough measure of how expensive it is to read one coefficient from * this expression. */ +#endif InnerStrideAtCompileTime = internal::inner_stride_at_compile_time<Derived>::ret, OuterStrideAtCompileTime = internal::outer_stride_at_compile_time<Derived>::ret @@ -278,7 +282,8 @@ template<typename Derived> class DenseBase Derived& operator=(const ReturnByValue<OtherDerived>& func); #ifndef EIGEN_PARSED_BY_DOXYGEN - /** Copies \a other into *this without evaluating other. \returns a reference to *this. */ + /** Copies \a other into *this without evaluating other. \returns a reference to *this. + * \deprecated */ template<typename OtherDerived> EIGEN_DEVICE_FUNC Derived& lazyAssign(const DenseBase<OtherDerived>& other); @@ -287,8 +292,15 @@ template<typename Derived> class DenseBase EIGEN_DEVICE_FUNC CommaInitializer<Derived> operator<< (const Scalar& s); +#ifndef EIGEN_TEST_EVALUATORS template<unsigned int Added,unsigned int Removed> const Flagged<Derived, Added, Removed> flagged() const; +#else + // TODO flagged is temporarly disabled. It seems useless now + template<unsigned int Added,unsigned int Removed> + const Derived& flagged() const + { return derived(); } +#endif template<typename OtherDerived> EIGEN_DEVICE_FUNC @@ -387,7 +399,31 @@ template<typename Derived> class DenseBase // size types on MSVC. return typename internal::eval<Derived>::type(derived()); } + +#ifdef EIGEN_TEST_EVALUATORS + /** swaps *this with the expression \a other. + * + */ + template<typename OtherDerived> + EIGEN_DEVICE_FUNC + void swap(const DenseBase<OtherDerived>& other, + int = OtherDerived::ThisConstantIsPrivateInPlainObjectBase) + { + eigen_assert(rows()==other.rows() && cols()==other.cols()); + call_assignment(derived(), other.const_cast_derived(), internal::swap_assign_op<Scalar>()); + } + /** swaps *this with the matrix or array \a other. + * + */ + template<typename OtherDerived> + EIGEN_DEVICE_FUNC + void swap(PlainObjectBase<OtherDerived>& other) + { + eigen_assert(rows()==other.rows() && cols()==other.cols()); + call_assignment(derived(), other.derived(), internal::swap_assign_op<Scalar>()); + } +#else // EIGEN_TEST_EVALUATORS /** swaps *this with the expression \a other. * */ @@ -408,7 +444,7 @@ template<typename Derived> class DenseBase { SwapWrapper<Derived>(derived()).lazyAssign(other.derived()); } - +#endif // EIGEN_TEST_EVALUATORS EIGEN_DEVICE_FUNC inline const NestByValue<Derived> nestByValue() const; EIGEN_DEVICE_FUNC inline const ForceAlignedAccess<Derived> forceAlignedAccess() const; diff --git a/Eigen/src/Core/DenseCoeffsBase.h b/Eigen/src/Core/DenseCoeffsBase.h index 4e986e875..de31b8df2 100644 --- a/Eigen/src/Core/DenseCoeffsBase.h +++ b/Eigen/src/Core/DenseCoeffsBase.h @@ -97,8 +97,12 @@ class DenseCoeffsBase<Derived,ReadOnlyAccessors> : public EigenBase<Derived> EIGEN_STRONG_INLINE CoeffReturnType coeff(Index row, Index col) const { eigen_internal_assert(row >= 0 && row < rows() - && col >= 0 && col < cols()); + && col >= 0 && col < cols()); +#ifndef EIGEN_TEST_EVALUATORS return derived().coeff(row, col); +#else + return typename internal::evaluator<Derived>::type(derived()).coeff(row,col); +#endif } EIGEN_DEVICE_FUNC @@ -117,7 +121,7 @@ class DenseCoeffsBase<Derived,ReadOnlyAccessors> : public EigenBase<Derived> { eigen_assert(row >= 0 && row < rows() && col >= 0 && col < cols()); - return derived().coeff(row, col); + return coeff(row, col); } /** Short version: don't use this function, use @@ -140,7 +144,11 @@ class DenseCoeffsBase<Derived,ReadOnlyAccessors> : public EigenBase<Derived> coeff(Index index) const { eigen_internal_assert(index >= 0 && index < size()); +#ifndef EIGEN_TEST_EVALUATORS return derived().coeff(index); +#else + return typename internal::evaluator<Derived>::type(derived()).coeff(index); +#endif } @@ -159,7 +167,7 @@ class DenseCoeffsBase<Derived,ReadOnlyAccessors> : public EigenBase<Derived> EIGEN_STATIC_ASSERT(Derived::IsVectorAtCompileTime, THE_BRACKET_OPERATOR_IS_ONLY_FOR_VECTORS__USE_THE_PARENTHESIS_OPERATOR_INSTEAD) eigen_assert(index >= 0 && index < size()); - return derived().coeff(index); + return coeff(index); } /** \returns the coefficient at given index. @@ -177,7 +185,7 @@ class DenseCoeffsBase<Derived,ReadOnlyAccessors> : public EigenBase<Derived> operator()(Index index) const { eigen_assert(index >= 0 && index < size()); - return derived().coeff(index); + return coeff(index); } /** equivalent to operator[](0). */ @@ -217,9 +225,12 @@ class DenseCoeffsBase<Derived,ReadOnlyAccessors> : public EigenBase<Derived> template<int LoadMode> EIGEN_STRONG_INLINE PacketReturnType packet(Index row, Index col) const { - eigen_internal_assert(row >= 0 && row < rows() - && col >= 0 && col < cols()); + eigen_internal_assert(row >= 0 && row < rows() && col >= 0 && col < cols()); +#ifndef EIGEN_TEST_EVALUATORS return derived().template packet<LoadMode>(row,col); +#else + return typename internal::evaluator<Derived>::type(derived()).template packet<LoadMode>(row,col); +#endif } @@ -245,7 +256,11 @@ class DenseCoeffsBase<Derived,ReadOnlyAccessors> : public EigenBase<Derived> EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const { eigen_internal_assert(index >= 0 && index < size()); +#ifndef EIGEN_TEST_EVALUATORS return derived().template packet<LoadMode>(index); +#else + return typename internal::evaluator<Derived>::type(derived()).template packet<LoadMode>(index); +#endif } protected: @@ -325,8 +340,12 @@ class DenseCoeffsBase<Derived, WriteAccessors> : public DenseCoeffsBase<Derived, EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { eigen_internal_assert(row >= 0 && row < rows() - && col >= 0 && col < cols()); + && col >= 0 && col < cols()); +#ifndef EIGEN_TEST_EVALUATORS return derived().coeffRef(row, col); +#else + return typename internal::evaluator<Derived>::type(derived()).coeffRef(row,col); +#endif } EIGEN_DEVICE_FUNC @@ -348,7 +367,7 @@ class DenseCoeffsBase<Derived, WriteAccessors> : public DenseCoeffsBase<Derived, { eigen_assert(row >= 0 && row < rows() && col >= 0 && col < cols()); - return derived().coeffRef(row, col); + return coeffRef(row, col); } @@ -372,7 +391,11 @@ class DenseCoeffsBase<Derived, WriteAccessors> : public DenseCoeffsBase<Derived, coeffRef(Index index) { eigen_internal_assert(index >= 0 && index < size()); +#ifndef EIGEN_TEST_EVALUATORS return derived().coeffRef(index); +#else + return typename internal::evaluator<Derived>::type(derived()).coeffRef(index); +#endif } /** \returns a reference to the coefficient at given index. @@ -389,7 +412,7 @@ class DenseCoeffsBase<Derived, WriteAccessors> : public DenseCoeffsBase<Derived, EIGEN_STATIC_ASSERT(Derived::IsVectorAtCompileTime, THE_BRACKET_OPERATOR_IS_ONLY_FOR_VECTORS__USE_THE_PARENTHESIS_OPERATOR_INSTEAD) eigen_assert(index >= 0 && index < size()); - return derived().coeffRef(index); + return coeffRef(index); } /** \returns a reference to the coefficient at given index. @@ -406,7 +429,7 @@ class DenseCoeffsBase<Derived, WriteAccessors> : public DenseCoeffsBase<Derived, operator()(Index index) { eigen_assert(index >= 0 && index < size()); - return derived().coeffRef(index); + return coeffRef(index); } /** equivalent to operator[](0). */ @@ -433,6 +456,7 @@ class DenseCoeffsBase<Derived, WriteAccessors> : public DenseCoeffsBase<Derived, EIGEN_STRONG_INLINE Scalar& w() { return (*this)[3]; } +#ifndef EIGEN_TEST_EVALUATORS /** \internal * Stores the given packet of coefficients, at the given row and column of this expression. It is your responsibility * to ensure that a packet really starts there. This method is only available on expressions having the @@ -569,6 +593,7 @@ class DenseCoeffsBase<Derived, WriteAccessors> : public DenseCoeffsBase<Derived, derived().template copyPacket< OtherDerived, StoreMode, LoadMode>(row, col, other); } #endif +#endif // EIGEN_TEST_EVALUATORS }; diff --git a/Eigen/src/Core/Diagonal.h b/Eigen/src/Core/Diagonal.h index b160479ab..3ff6a3e66 100644 --- a/Eigen/src/Core/Diagonal.h +++ b/Eigen/src/Core/Diagonal.h @@ -51,9 +51,14 @@ struct traits<Diagonal<MatrixType,DiagIndex> > : (EIGEN_PLAIN_ENUM_MIN(MatrixType::MaxRowsAtCompileTime - EIGEN_PLAIN_ENUM_MAX(-DiagIndex, 0), MatrixType::MaxColsAtCompileTime - EIGEN_PLAIN_ENUM_MAX( DiagIndex, 0))), MaxColsAtCompileTime = 1, +#ifndef EIGEN_TEST_EVALUATORS MaskLvalueBit = is_lvalue<MatrixType>::value ? LvalueBit : 0, Flags = (unsigned int)_MatrixTypeNested::Flags & (HereditaryBits | LinearAccessBit | MaskLvalueBit | DirectAccessBit) & ~RowMajorBit, CoeffReadCost = _MatrixTypeNested::CoeffReadCost, +#else + MaskLvalueBit = is_lvalue<MatrixType>::value ? LvalueBit : 0, + Flags = (unsigned int)_MatrixTypeNested::Flags & (RowMajorBit | MaskLvalueBit | DirectAccessBit) & ~RowMajorBit, // FIXME DirectAccessBit should not be handled by expressions +#endif MatrixTypeOuterStride = outer_stride_at_compile_time<MatrixType>::ret, InnerStrideAtCompileTime = MatrixTypeOuterStride == Dynamic ? Dynamic : MatrixTypeOuterStride+1, OuterStrideAtCompileTime = 0 diff --git a/Eigen/src/Core/DiagonalMatrix.h b/Eigen/src/Core/DiagonalMatrix.h index 96b65483d..801131b54 100644 --- a/Eigen/src/Core/DiagonalMatrix.h +++ b/Eigen/src/Core/DiagonalMatrix.h @@ -30,7 +30,7 @@ class DiagonalBase : public EigenBase<Derived> MaxRowsAtCompileTime = DiagonalVectorType::MaxSizeAtCompileTime, MaxColsAtCompileTime = DiagonalVectorType::MaxSizeAtCompileTime, IsVectorAtCompileTime = 0, - Flags = 0 + Flags = NoPreferredStorageOrderBit }; typedef Matrix<Scalar, RowsAtCompileTime, ColsAtCompileTime, 0, MaxRowsAtCompileTime, MaxColsAtCompileTime> DenseMatrixType; @@ -44,6 +44,8 @@ class DiagonalBase : public EigenBase<Derived> EIGEN_DEVICE_FUNC DenseMatrixType toDenseMatrix() const { return derived(); } + +#ifndef EIGEN_TEST_EVALUATORS template<typename DenseDerived> EIGEN_DEVICE_FUNC void evalTo(MatrixBase<DenseDerived> &other) const; @@ -55,6 +57,7 @@ class DiagonalBase : public EigenBase<Derived> EIGEN_DEVICE_FUNC void subTo(MatrixBase<DenseDerived> &other) const { other.diagonal() -= diagonal(); } +#endif // EIGEN_TEST_EVALUATORS EIGEN_DEVICE_FUNC inline const DiagonalVectorType& diagonal() const { return derived().diagonal(); } @@ -66,6 +69,7 @@ class DiagonalBase : public EigenBase<Derived> EIGEN_DEVICE_FUNC inline Index cols() const { return diagonal().size(); } +#ifndef EIGEN_TEST_EVALUATORS /** \returns the diagonal matrix product of \c *this by the matrix \a matrix. */ template<typename MatrixDerived> @@ -75,6 +79,15 @@ class DiagonalBase : public EigenBase<Derived> { return DiagonalProduct<MatrixDerived, Derived, OnTheLeft>(matrix.derived(), derived()); } +#else + template<typename MatrixDerived> + EIGEN_DEVICE_FUNC + const Product<Derived,MatrixDerived,LazyProduct> + operator*(const MatrixBase<MatrixDerived> &matrix) const + { + return Product<Derived, MatrixDerived, LazyProduct>(derived(),matrix.derived()); + } +#endif // EIGEN_TEST_EVALUATORS EIGEN_DEVICE_FUNC inline const DiagonalWrapper<const CwiseUnaryOp<internal::scalar_inverse_op<Scalar>, const DiagonalVectorType> > @@ -97,6 +110,7 @@ class DiagonalBase : public EigenBase<Derived> } }; +#ifndef EIGEN_TEST_EVALUATORS template<typename Derived> template<typename DenseDerived> void DiagonalBase<Derived>::evalTo(MatrixBase<DenseDerived> &other) const @@ -104,6 +118,8 @@ void DiagonalBase<Derived>::evalTo(MatrixBase<DenseDerived> &other) const other.setZero(); other.diagonal() = diagonal(); } +#endif // EIGEN_TEST_EVALUATORS + #endif /** \class DiagonalMatrix @@ -125,10 +141,10 @@ struct traits<DiagonalMatrix<_Scalar,SizeAtCompileTime,MaxSizeAtCompileTime> > : traits<Matrix<_Scalar,SizeAtCompileTime,SizeAtCompileTime,0,MaxSizeAtCompileTime,MaxSizeAtCompileTime> > { typedef Matrix<_Scalar,SizeAtCompileTime,1,0,MaxSizeAtCompileTime,1> DiagonalVectorType; - typedef Dense StorageKind; + typedef DiagonalShape StorageKind; typedef DenseIndex Index; enum { - Flags = LvalueBit + Flags = LvalueBit | NoPreferredStorageOrderBit }; }; } @@ -249,13 +265,18 @@ struct traits<DiagonalWrapper<_DiagonalVectorType> > typedef _DiagonalVectorType DiagonalVectorType; typedef typename DiagonalVectorType::Scalar Scalar; typedef typename DiagonalVectorType::Index Index; - typedef typename DiagonalVectorType::StorageKind StorageKind; + typedef DiagonalShape StorageKind; + typedef typename traits<DiagonalVectorType>::XprKind XprKind; enum { RowsAtCompileTime = DiagonalVectorType::SizeAtCompileTime, ColsAtCompileTime = DiagonalVectorType::SizeAtCompileTime, - MaxRowsAtCompileTime = DiagonalVectorType::SizeAtCompileTime, - MaxColsAtCompileTime = DiagonalVectorType::SizeAtCompileTime, - Flags = traits<DiagonalVectorType>::Flags & LvalueBit + MaxRowsAtCompileTime = DiagonalVectorType::MaxSizeAtCompileTime, + MaxColsAtCompileTime = DiagonalVectorType::MaxSizeAtCompileTime, + Flags = (traits<DiagonalVectorType>::Flags & LvalueBit) | NoPreferredStorageOrderBit +#ifndef EIGEN_TEST_EVALUATORS + , + CoeffReadCost = traits<_DiagonalVectorType>::CoeffReadCost +#endif }; }; } @@ -326,6 +347,29 @@ bool MatrixBase<Derived>::isDiagonal(const RealScalar& prec) const return true; } +#ifdef EIGEN_ENABLE_EVALUATORS +namespace internal { + +template<> struct storage_kind_to_shape<DiagonalShape> { typedef DiagonalShape Shape; }; + +struct Diagonal2Dense {}; + +template<> struct AssignmentKind<DenseShape,DiagonalShape> { typedef Diagonal2Dense Kind; }; + +// Diagonal matrix to Dense assignment +template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> +struct Assignment<DstXprType, SrcXprType, Functor, Diagonal2Dense, Scalar> +{ + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar> &/*func*/) + { + dst.setZero(); + dst.diagonal() = src.diagonal(); + } +}; + +} // namespace internal +#endif // EIGEN_ENABLE_EVALUATORS + } // end namespace Eigen #endif // EIGEN_DIAGONALMATRIX_H diff --git a/Eigen/src/Core/DiagonalProduct.h b/Eigen/src/Core/DiagonalProduct.h index c03a0c2e1..c6dafdddc 100644 --- a/Eigen/src/Core/DiagonalProduct.h +++ b/Eigen/src/Core/DiagonalProduct.h @@ -13,6 +13,7 @@ namespace Eigen { +#ifndef EIGEN_TEST_EVALUATORS namespace internal { template<typename MatrixType, typename DiagonalType, int ProductOrder> struct traits<DiagonalProduct<MatrixType, DiagonalType, ProductOrder> > @@ -25,6 +26,7 @@ struct traits<DiagonalProduct<MatrixType, DiagonalType, ProductOrder> > MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime, +#ifndef EIGEN_TEST_EVALUATORS _StorageOrder = MatrixType::Flags & RowMajorBit ? RowMajor : ColMajor, _ScalarAccessOnDiag = !((int(_StorageOrder) == ColMajor && int(ProductOrder) == OnTheLeft) ||(int(_StorageOrder) == RowMajor && int(ProductOrder) == OnTheRight)), @@ -33,9 +35,11 @@ struct traits<DiagonalProduct<MatrixType, DiagonalType, ProductOrder> > //_Vectorizable = bool(int(MatrixType::Flags)&PacketAccessBit) && ((!_PacketOnDiag) || (_SameTypes && bool(int(DiagonalType::DiagonalVectorType::Flags)&PacketAccessBit))), _Vectorizable = bool(int(MatrixType::Flags)&PacketAccessBit) && _SameTypes && (_ScalarAccessOnDiag || (bool(int(DiagonalType::DiagonalVectorType::Flags)&PacketAccessBit))), _LinearAccessMask = (RowsAtCompileTime==1 || ColsAtCompileTime==1) ? LinearAccessBit : 0, - - Flags = ((HereditaryBits|_LinearAccessMask) & (unsigned int)(MatrixType::Flags)) | (_Vectorizable ? PacketAccessBit : 0) | AlignedBit,//(int(MatrixType::Flags)&int(DiagonalType::DiagonalVectorType::Flags)&AlignedBit), + Flags = ((HereditaryBits|_LinearAccessMask) & (unsigned int)(MatrixType::Flags)) | (_Vectorizable ? PacketAccessBit : 0) | AlignedBit, //(int(MatrixType::Flags)&int(DiagonalType::DiagonalVectorType::Flags)&AlignedBit), CoeffReadCost = NumTraits<Scalar>::MulCost + MatrixType::CoeffReadCost + DiagonalType::DiagonalVectorType::CoeffReadCost +#else + Flags = RowMajorBit & (unsigned int)(MatrixType::Flags) +#endif }; }; } @@ -124,6 +128,17 @@ MatrixBase<Derived>::operator*(const DiagonalBase<DiagonalDerived> &a_diagonal) { return DiagonalProduct<Derived, DiagonalDerived, OnTheRight>(derived(), a_diagonal.derived()); } +#else // EIGEN_TEST_EVALUATORS +/** \returns the diagonal matrix product of \c *this by the diagonal matrix \a diagonal. + */ +template<typename Derived> +template<typename DiagonalDerived> +inline const Product<Derived, DiagonalDerived, LazyProduct> +MatrixBase<Derived>::operator*(const DiagonalBase<DiagonalDerived> &a_diagonal) const +{ + return Product<Derived, DiagonalDerived, LazyProduct>(derived(),a_diagonal.derived()); +} +#endif // EIGEN_TEST_EVALUATORS } // end namespace Eigen diff --git a/Eigen/src/Core/Dot.h b/Eigen/src/Core/Dot.h index db16e4acc..d6441c6a5 100644 --- a/Eigen/src/Core/Dot.h +++ b/Eigen/src/Core/Dot.h @@ -113,8 +113,13 @@ template<typename Derived> inline const typename MatrixBase<Derived>::PlainObject MatrixBase<Derived>::normalized() const { - typedef typename internal::nested<Derived>::type Nested; +#ifndef EIGEN_TEST_EVALUATORS + typedef typename internal::nested<Derived,2>::type Nested; typedef typename internal::remove_reference<Nested>::type _Nested; +#else + typedef typename internal::nested_eval<Derived,2>::type _Nested; +// typedef typename internal::remove_reference<Nested>::type _Nested; +#endif // EIGEN_TEST_EVALUATORS _Nested n(derived()); return n / n.norm(); } @@ -206,8 +211,13 @@ template<typename OtherDerived> bool MatrixBase<Derived>::isOrthogonal (const MatrixBase<OtherDerived>& other, const RealScalar& prec) const { +#ifndef EIGEN_TEST_EVALUATORS typename internal::nested<Derived,2>::type nested(derived()); typename internal::nested<OtherDerived,2>::type otherNested(other.derived()); +#else + typename internal::nested_eval<Derived,2>::type nested(derived()); + typename internal::nested_eval<OtherDerived,2>::type otherNested(other.derived()); +#endif return numext::abs2(nested.dot(otherNested)) <= prec * prec * nested.squaredNorm() * otherNested.squaredNorm(); } diff --git a/Eigen/src/Core/EigenBase.h b/Eigen/src/Core/EigenBase.h index 1a577c2dc..986e2a196 100644 --- a/Eigen/src/Core/EigenBase.h +++ b/Eigen/src/Core/EigenBase.h @@ -121,7 +121,11 @@ template<typename Derived> template<typename OtherDerived> Derived& DenseBase<Derived>::operator=(const EigenBase<OtherDerived> &other) { +#ifndef EIGEN_TEST_EVALUATORS other.derived().evalTo(derived()); +#else + call_assignment(derived(), other.derived()); +#endif return derived(); } @@ -129,7 +133,11 @@ template<typename Derived> template<typename OtherDerived> Derived& DenseBase<Derived>::operator+=(const EigenBase<OtherDerived> &other) { +#ifndef EIGEN_TEST_EVALUATORS other.derived().addTo(derived()); +#else + call_assignment(derived(), other.derived(), internal::add_assign_op<Scalar>()); +#endif return derived(); } @@ -137,7 +145,11 @@ template<typename Derived> template<typename OtherDerived> Derived& DenseBase<Derived>::operator-=(const EigenBase<OtherDerived> &other) { +#ifndef EIGEN_TEST_EVALUATORS other.derived().subTo(derived()); +#else + call_assignment(derived(), other.derived(), internal::sub_assign_op<Scalar>()); +#endif return derived(); } diff --git a/Eigen/src/Core/Fuzzy.h b/Eigen/src/Core/Fuzzy.h index f9a88dd3c..9c8d10683 100644 --- a/Eigen/src/Core/Fuzzy.h +++ b/Eigen/src/Core/Fuzzy.h @@ -23,8 +23,13 @@ struct isApprox_selector static bool run(const Derived& x, const OtherDerived& y, const typename Derived::RealScalar& prec) { EIGEN_USING_STD_MATH(min); +#ifdef EIGEN_TEST_EVALUATORS + typename internal::nested_eval<Derived,2>::type nested(x); + typename internal::nested_eval<OtherDerived,2>::type otherNested(y); +#else typename internal::nested<Derived,2>::type nested(x); typename internal::nested<OtherDerived,2>::type otherNested(y); +#endif return (nested - otherNested).cwiseAbs2().sum() <= prec * prec * (min)(nested.cwiseAbs2().sum(), otherNested.cwiseAbs2().sum()); } }; diff --git a/Eigen/src/Core/GeneralProduct.h b/Eigen/src/Core/GeneralProduct.h index 624b8b6e8..76271a87d 100644 --- a/Eigen/src/Core/GeneralProduct.h +++ b/Eigen/src/Core/GeneralProduct.h @@ -13,6 +13,7 @@ namespace Eigen { +#ifndef EIGEN_TEST_EVALUATORS /** \class GeneralProduct * \ingroup Core_Module * @@ -34,6 +35,8 @@ namespace Eigen { */ template<typename Lhs, typename Rhs, int ProductType = internal::product_type<Lhs,Rhs>::value> class GeneralProduct; +#endif // EIGEN_TEST_EVALUATORS + enum { Large = 2, @@ -59,14 +62,14 @@ template<typename Lhs, typename Rhs> struct product_type typedef typename remove_all<Lhs>::type _Lhs; typedef typename remove_all<Rhs>::type _Rhs; enum { - MaxRows = _Lhs::MaxRowsAtCompileTime, - Rows = _Lhs::RowsAtCompileTime, - MaxCols = _Rhs::MaxColsAtCompileTime, - Cols = _Rhs::ColsAtCompileTime, - MaxDepth = EIGEN_SIZE_MIN_PREFER_FIXED(_Lhs::MaxColsAtCompileTime, - _Rhs::MaxRowsAtCompileTime), - Depth = EIGEN_SIZE_MIN_PREFER_FIXED(_Lhs::ColsAtCompileTime, - _Rhs::RowsAtCompileTime) + MaxRows = traits<_Lhs>::MaxRowsAtCompileTime, + Rows = traits<_Lhs>::RowsAtCompileTime, + MaxCols = traits<_Rhs>::MaxColsAtCompileTime, + Cols = traits<_Rhs>::ColsAtCompileTime, + MaxDepth = EIGEN_SIZE_MIN_PREFER_FIXED(traits<_Lhs>::MaxColsAtCompileTime, + traits<_Rhs>::MaxRowsAtCompileTime), + Depth = EIGEN_SIZE_MIN_PREFER_FIXED(traits<_Lhs>::ColsAtCompileTime, + traits<_Rhs>::RowsAtCompileTime) }; // the splitting into different lines of code here, introducing the _select enums and the typedef below, @@ -81,7 +84,8 @@ private: public: enum { - value = selector::ret + value = selector::ret, + ret = selector::ret }; #ifdef EIGEN_DEBUG_PRODUCT static void debug() @@ -97,6 +101,31 @@ public: #endif }; +// template<typename Lhs, typename Rhs> struct product_tag +// { +// private: +// +// typedef typename remove_all<Lhs>::type _Lhs; +// typedef typename remove_all<Rhs>::type _Rhs; +// enum { +// Rows = _Lhs::RowsAtCompileTime, +// Cols = _Rhs::ColsAtCompileTime, +// Depth = EIGEN_SIZE_MIN_PREFER_FIXED(_Lhs::ColsAtCompileTime, _Rhs::RowsAtCompileTime) +// }; +// +// enum { +// rows_select = Rows==1 ? int(Rows) : int(Large), +// cols_select = Cols==1 ? int(Cols) : int(Large), +// depth_select = Depth==1 ? int(Depth) : int(Large) +// }; +// typedef product_type_selector<rows_select, cols_select, depth_select> selector; +// +// public: +// enum { +// ret = selector::ret +// }; +// +// }; /* The following allows to select the kind of product at compile time * based on the three dimensions of the product. @@ -127,6 +156,7 @@ template<> struct product_type_selector<Large,Large,Small> { enum } // end namespace internal +#ifndef EIGEN_TEST_EVALUATORS /** \class ProductReturnType * \ingroup Core_Module * @@ -174,6 +204,7 @@ struct ProductReturnType<Lhs,Rhs,LazyCoeffBasedProductMode> template<typename Lhs, typename Rhs> struct LazyProductReturnType : public ProductReturnType<Lhs,Rhs,LazyCoeffBasedProductMode> {}; +#endif /*********************************************************************** * Implementation of Inner Vector Vector Product @@ -185,6 +216,7 @@ struct LazyProductReturnType : public ProductReturnType<Lhs,Rhs,LazyCoeffBasedPr // Cons: this could be a problem if in a meta unrolled algorithm a matrix-matrix // product ends up to a row-vector times col-vector product... To tackle this use // case, we could have a specialization for Block<MatrixType,1,1> with: operator=(Scalar x); +#ifndef EIGEN_TEST_EVALUATORS namespace internal { @@ -215,11 +247,12 @@ class GeneralProduct<Lhs, Rhs, InnerProduct> return Base::coeff(0,0); } }; - +#endif // EIGEN_TEST_EVALUATORS /*********************************************************************** * Implementation of Outer Vector Vector Product ***********************************************************************/ +#ifndef EIGEN_TEST_EVALUATORS namespace internal { // Column major @@ -299,6 +332,8 @@ class GeneralProduct<Lhs, Rhs, OuterProduct> } }; +#endif // EIGEN_TEST_EVALUATORS + /*********************************************************************** * Implementation of General Matrix Vector Product ***********************************************************************/ @@ -312,16 +347,22 @@ class GeneralProduct<Lhs, Rhs, OuterProduct> */ namespace internal { +#ifndef EIGEN_TEST_EVALUATORS template<typename Lhs, typename Rhs> struct traits<GeneralProduct<Lhs,Rhs,GemvProduct> > : traits<ProductBase<GeneralProduct<Lhs,Rhs,GemvProduct>, Lhs, Rhs> > {}; - template<int Side, int StorageOrder, bool BlasCompatible> struct gemv_selector; +#endif +#ifdef EIGEN_ENABLE_EVALUATORS +template<int Side, int StorageOrder, bool BlasCompatible> +struct gemv_dense_sense_selector; +#endif } // end namespace internal +#ifndef EIGEN_TEST_EVALUATORS template<typename Lhs, typename Rhs> class GeneralProduct<Lhs, Rhs, GemvProduct> : public ProductBase<GeneralProduct<Lhs,Rhs,GemvProduct>, Lhs, Rhs> @@ -348,24 +389,10 @@ class GeneralProduct<Lhs, Rhs, GemvProduct> bool(internal::blas_traits<MatrixType>::HasUsableDirectAccess)>::run(*this, dst, alpha); } }; +#endif namespace internal { -// The vector is on the left => transposition -template<int StorageOrder, bool BlasCompatible> -struct gemv_selector<OnTheLeft,StorageOrder,BlasCompatible> -{ - template<typename ProductType, typename Dest> - static void run(const ProductType& prod, Dest& dest, const typename ProductType::Scalar& alpha) - { - Transpose<Dest> destT(dest); - enum { OtherStorageOrder = StorageOrder == RowMajor ? ColMajor : RowMajor }; - gemv_selector<OnTheRight,OtherStorageOrder,BlasCompatible> - ::run(GeneralProduct<Transpose<const typename ProductType::_RhsNested>,Transpose<const typename ProductType::_LhsNested>, GemvProduct> - (prod.rhs().transpose(), prod.lhs().transpose()), destT, alpha); - } -}; - template<typename Scalar,int Size,int MaxSize,bool Cond> struct gemv_static_vector_if; template<typename Scalar,int Size,int MaxSize> @@ -402,6 +429,23 @@ struct gemv_static_vector_if<Scalar,Size,MaxSize,true> #endif }; +#ifndef EIGEN_TEST_EVALUATORS + +// The vector is on the left => transposition +template<int StorageOrder, bool BlasCompatible> +struct gemv_selector<OnTheLeft,StorageOrder,BlasCompatible> +{ + template<typename ProductType, typename Dest> + static void run(const ProductType& prod, Dest& dest, const typename ProductType::Scalar& alpha) + { + Transpose<Dest> destT(dest); + enum { OtherStorageOrder = StorageOrder == RowMajor ? ColMajor : RowMajor }; + gemv_selector<OnTheRight,OtherStorageOrder,BlasCompatible> + ::run(GeneralProduct<Transpose<const typename ProductType::_RhsNested>,Transpose<const typename ProductType::_LhsNested>, GemvProduct> + (prod.rhs().transpose(), prod.lhs().transpose()), destT, alpha); + } +}; + template<> struct gemv_selector<OnTheRight,ColMajor,true> { template<typename ProductType, typename Dest> @@ -552,6 +596,179 @@ template<> struct gemv_selector<OnTheRight,RowMajor,false> } }; +#endif // EIGEN_TEST_EVALUATORS + +#ifdef EIGEN_ENABLE_EVALUATORS + +// The vector is on the left => transposition +template<int StorageOrder, bool BlasCompatible> +struct gemv_dense_sense_selector<OnTheLeft,StorageOrder,BlasCompatible> +{ + template<typename Lhs, typename Rhs, typename Dest> + static void run(const Lhs &lhs, const Rhs &rhs, Dest& dest, const typename Dest::Scalar& alpha) + { + Transpose<Dest> destT(dest); + enum { OtherStorageOrder = StorageOrder == RowMajor ? ColMajor : RowMajor }; + gemv_dense_sense_selector<OnTheRight,OtherStorageOrder,BlasCompatible> + ::run(rhs.transpose(), lhs.transpose(), destT, alpha); + } +}; + +template<> struct gemv_dense_sense_selector<OnTheRight,ColMajor,true> +{ + template<typename Lhs, typename Rhs, typename Dest> + static inline void run(const Lhs &lhs, const Rhs &rhs, Dest& dest, const typename Dest::Scalar& alpha) + { + typedef typename Dest::Index Index; + typedef typename Lhs::Scalar LhsScalar; + typedef typename Rhs::Scalar RhsScalar; + typedef typename Dest::Scalar ResScalar; + typedef typename Dest::RealScalar RealScalar; + + typedef internal::blas_traits<Lhs> LhsBlasTraits; + typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; + typedef internal::blas_traits<Rhs> RhsBlasTraits; + typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; + + typedef Map<Matrix<ResScalar,Dynamic,1>, Aligned> MappedDest; + + ActualLhsType actualLhs = LhsBlasTraits::extract(lhs); + ActualRhsType actualRhs = RhsBlasTraits::extract(rhs); + + ResScalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(lhs) + * RhsBlasTraits::extractScalarFactor(rhs); + + enum { + // FIXME find a way to allow an inner stride on the result if packet_traits<Scalar>::size==1 + // on, the other hand it is good for the cache to pack the vector anyways... + EvalToDestAtCompileTime = Dest::InnerStrideAtCompileTime==1, + ComplexByReal = (NumTraits<LhsScalar>::IsComplex) && (!NumTraits<RhsScalar>::IsComplex), + MightCannotUseDest = (Dest::InnerStrideAtCompileTime!=1) || ComplexByReal + }; + + gemv_static_vector_if<ResScalar,Dest::SizeAtCompileTime,Dest::MaxSizeAtCompileTime,MightCannotUseDest> static_dest; + + bool alphaIsCompatible = (!ComplexByReal) || (numext::imag(actualAlpha)==RealScalar(0)); + bool evalToDest = EvalToDestAtCompileTime && alphaIsCompatible; + + RhsScalar compatibleAlpha = get_factor<ResScalar,RhsScalar>::run(actualAlpha); + + ei_declare_aligned_stack_constructed_variable(ResScalar,actualDestPtr,dest.size(), + evalToDest ? dest.data() : static_dest.data()); + + if(!evalToDest) + { + #ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN + int size = dest.size(); + EIGEN_DENSE_STORAGE_CTOR_PLUGIN + #endif + if(!alphaIsCompatible) + { + MappedDest(actualDestPtr, dest.size()).setZero(); + compatibleAlpha = RhsScalar(1); + } + else + MappedDest(actualDestPtr, dest.size()) = dest; + } + + general_matrix_vector_product + <Index,LhsScalar,ColMajor,LhsBlasTraits::NeedToConjugate,RhsScalar,RhsBlasTraits::NeedToConjugate>::run( + actualLhs.rows(), actualLhs.cols(), + actualLhs.data(), actualLhs.outerStride(), + actualRhs.data(), actualRhs.innerStride(), + actualDestPtr, 1, + compatibleAlpha); + + if (!evalToDest) + { + if(!alphaIsCompatible) + dest += actualAlpha * MappedDest(actualDestPtr, dest.size()); + else + dest = MappedDest(actualDestPtr, dest.size()); + } + } +}; + +template<> struct gemv_dense_sense_selector<OnTheRight,RowMajor,true> +{ + template<typename Lhs, typename Rhs, typename Dest> + static void run(const Lhs &lhs, const Rhs &rhs, Dest& dest, const typename Dest::Scalar& alpha) + { + typedef typename Dest::Index Index; + typedef typename Lhs::Scalar LhsScalar; + typedef typename Rhs::Scalar RhsScalar; + typedef typename Dest::Scalar ResScalar; + + typedef internal::blas_traits<Lhs> LhsBlasTraits; + typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; + typedef internal::blas_traits<Rhs> RhsBlasTraits; + typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; + typedef typename internal::remove_all<ActualRhsType>::type ActualRhsTypeCleaned; + + typename add_const<ActualLhsType>::type actualLhs = LhsBlasTraits::extract(lhs); + typename add_const<ActualRhsType>::type actualRhs = RhsBlasTraits::extract(rhs); + + ResScalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(lhs) + * RhsBlasTraits::extractScalarFactor(rhs); + + enum { + // FIXME find a way to allow an inner stride on the result if packet_traits<Scalar>::size==1 + // on, the other hand it is good for the cache to pack the vector anyways... + DirectlyUseRhs = ActualRhsTypeCleaned::InnerStrideAtCompileTime==1 + }; + + gemv_static_vector_if<RhsScalar,ActualRhsTypeCleaned::SizeAtCompileTime,ActualRhsTypeCleaned::MaxSizeAtCompileTime,!DirectlyUseRhs> static_rhs; + + ei_declare_aligned_stack_constructed_variable(RhsScalar,actualRhsPtr,actualRhs.size(), + DirectlyUseRhs ? const_cast<RhsScalar*>(actualRhs.data()) : static_rhs.data()); + + if(!DirectlyUseRhs) + { + #ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN + int size = actualRhs.size(); + EIGEN_DENSE_STORAGE_CTOR_PLUGIN + #endif + Map<typename ActualRhsTypeCleaned::PlainObject>(actualRhsPtr, actualRhs.size()) = actualRhs; + } + + general_matrix_vector_product + <Index,LhsScalar,RowMajor,LhsBlasTraits::NeedToConjugate,RhsScalar,RhsBlasTraits::NeedToConjugate>::run( + actualLhs.rows(), actualLhs.cols(), + actualLhs.data(), actualLhs.outerStride(), + actualRhsPtr, 1, + dest.data(), dest.innerStride(), + actualAlpha); + } +}; + +template<> struct gemv_dense_sense_selector<OnTheRight,ColMajor,false> +{ + template<typename Lhs, typename Rhs, typename Dest> + static void run(const Lhs &lhs, const Rhs &rhs, Dest& dest, const typename Dest::Scalar& alpha) + { + typedef typename Dest::Index Index; + // TODO makes sure dest is sequentially stored in memory, otherwise use a temp + const Index size = rhs.rows(); + for(Index k=0; k<size; ++k) + dest += (alpha*rhs.coeff(k)) * lhs.col(k); + } +}; + +template<> struct gemv_dense_sense_selector<OnTheRight,RowMajor,false> +{ + template<typename Lhs, typename Rhs, typename Dest> + static void run(const Lhs &lhs, const Rhs &rhs, Dest& dest, const typename Dest::Scalar& alpha) + { + typedef typename Dest::Index Index; + // TODO makes sure rhs is sequentially stored in memory, otherwise use a temp + const Index rows = dest.rows(); + for(Index i=0; i<rows; ++i) + dest.coeffRef(i) += alpha * (lhs.row(i).cwiseProduct(rhs.transpose())).sum(); + } +}; + +#endif // EIGEN_ENABLE_EVALUATORS + } // end namespace internal /*************************************************************************** @@ -597,7 +814,7 @@ MatrixBase<Derived>::operator*(const MatrixBase<OtherDerived> &other) const return Product<Derived, OtherDerived>(derived(), other.derived()); } -#else +#else // EIGEN_TEST_EVALUATORS template<typename Derived> template<typename OtherDerived> inline const typename ProductReturnType<Derived, OtherDerived>::Type @@ -627,9 +844,10 @@ MatrixBase<Derived>::operator*(const MatrixBase<OtherDerived> &other) const #endif return typename ProductReturnType<Derived,OtherDerived>::Type(derived(), other.derived()); } -#endif +#endif // EIGEN_TEST_EVALUATORS + +#endif // __CUDACC__ -#endif /** \returns an expression of the matrix product of \c *this and \a other without implicit evaluation. * * The returned product will behave like any other expressions: the coefficients of the product will be @@ -641,6 +859,31 @@ MatrixBase<Derived>::operator*(const MatrixBase<OtherDerived> &other) const * * \sa operator*(const MatrixBase&) */ +#ifdef EIGEN_TEST_EVALUATORS +template<typename Derived> +template<typename OtherDerived> +const Product<Derived,OtherDerived,LazyProduct> +MatrixBase<Derived>::lazyProduct(const MatrixBase<OtherDerived> &other) const +{ + enum { + ProductIsValid = Derived::ColsAtCompileTime==Dynamic + || OtherDerived::RowsAtCompileTime==Dynamic + || int(Derived::ColsAtCompileTime)==int(OtherDerived::RowsAtCompileTime), + AreVectors = Derived::IsVectorAtCompileTime && OtherDerived::IsVectorAtCompileTime, + SameSizes = EIGEN_PREDICATE_SAME_MATRIX_SIZE(Derived,OtherDerived) + }; + // note to the lost user: + // * for a dot product use: v1.dot(v2) + // * for a coeff-wise product use: v1.cwiseProduct(v2) + EIGEN_STATIC_ASSERT(ProductIsValid || !(AreVectors && SameSizes), + INVALID_VECTOR_VECTOR_PRODUCT__IF_YOU_WANTED_A_DOT_OR_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTIONS) + EIGEN_STATIC_ASSERT(ProductIsValid || !(SameSizes && !AreVectors), + INVALID_MATRIX_PRODUCT__IF_YOU_WANTED_A_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTION) + EIGEN_STATIC_ASSERT(ProductIsValid || SameSizes, INVALID_MATRIX_PRODUCT) + + return Product<Derived,OtherDerived,LazyProduct>(derived(), other.derived()); +} +#else // EIGEN_TEST_EVALUATORS template<typename Derived> template<typename OtherDerived> const typename LazyProductReturnType<Derived,OtherDerived>::Type @@ -664,6 +907,7 @@ MatrixBase<Derived>::lazyProduct(const MatrixBase<OtherDerived> &other) const return typename LazyProductReturnType<Derived,OtherDerived>::Type(derived(), other.derived()); } +#endif // EIGEN_TEST_EVALUATORS } // end namespace Eigen diff --git a/Eigen/src/Core/Inverse.h b/Eigen/src/Core/Inverse.h new file mode 100644 index 000000000..1d167867f --- /dev/null +++ b/Eigen/src/Core/Inverse.h @@ -0,0 +1,134 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_INVERSE_H +#define EIGEN_INVERSE_H + +namespace Eigen { + +#ifdef EIGEN_TEST_EVALUATORS + +// TODO move the general declaration in Core, and rename this file DenseInverseImpl.h, or something like this... + +template<typename XprType,typename StorageKind> class InverseImpl; + +namespace internal { + +template<typename XprType> +struct traits<Inverse<XprType> > + : traits<typename XprType::PlainObject> +{ + typedef typename XprType::PlainObject PlainObject; + typedef traits<PlainObject> BaseTraits; + enum { + Flags = BaseTraits::Flags & RowMajorBit, + CoeffReadCost = Dynamic + }; +}; + +} // end namespace internal + +/** \class Inverse + * + * \brief Expression of the inverse of another expression + * + * \tparam XprType the type of the expression we are taking the inverse + * + * This class represents an abstract expression of A.inverse() + * and most of the time this is the only way it is used. + * + */ +template<typename XprType> +class Inverse : public InverseImpl<XprType,typename internal::traits<XprType>::StorageKind> +{ +public: + typedef typename XprType::Index Index; + typedef typename XprType::PlainObject PlainObject; + typedef typename internal::nested<XprType>::type XprTypeNested; + typedef typename internal::remove_all<XprTypeNested>::type XprTypeNestedCleaned; + + Inverse(const XprType &xpr) + : m_xpr(xpr) + {} + + EIGEN_DEVICE_FUNC Index rows() const { return m_xpr.rows(); } + EIGEN_DEVICE_FUNC Index cols() const { return m_xpr.cols(); } + + EIGEN_DEVICE_FUNC const XprTypeNestedCleaned& nestedExpression() const { return m_xpr; } + +protected: + XprTypeNested m_xpr; +}; + +/** \internal + * Specialization of the Inverse expression for dense expressions. + * Direct access to the coefficients are discared. + * FIXME this intermediate class is probably not needed anymore. + */ +template<typename XprType> +class InverseImpl<XprType,Dense> + : public MatrixBase<Inverse<XprType> > +{ + typedef Inverse<XprType> Derived; + +public: + + typedef MatrixBase<Derived> Base; + EIGEN_DENSE_PUBLIC_INTERFACE(Derived) + typedef typename internal::remove_all<XprType>::type NestedExpression; + +private: + + Scalar coeff(Index row, Index col) const; + Scalar coeff(Index i) const; +}; + +namespace internal { + +/** \internal + * \brief Default evaluator for Inverse expression. + * + * This default evaluator for Inverse expression simply evaluate the inverse into a temporary + * by a call to internal::call_assignment_no_alias. + * Therefore, inverse implementers only have to specialize Assignment<Dst,Inverse<...>, ...> for + * there own nested expression. + * + * \sa class Inverse + */ +template<typename XprType> +struct unary_evaluator<Inverse<XprType> > + : public evaluator<typename Inverse<XprType>::PlainObject>::type +{ + typedef Inverse<XprType> InverseType; + typedef typename InverseType::PlainObject PlainObject; + typedef typename evaluator<PlainObject>::type Base; + + typedef evaluator<InverseType> type; + typedef evaluator<InverseType> nestedType; + + enum { Flags = Base::Flags | EvalBeforeNestingBit }; + + unary_evaluator(const InverseType& inv_xpr) + : m_result(inv_xpr.rows(), inv_xpr.cols()) + { + ::new (static_cast<Base*>(this)) Base(m_result); + internal::call_assignment_no_alias(m_result, inv_xpr); + } + +protected: + PlainObject m_result; +}; + +} // end namespace internal + +#endif + +} // end namespace Eigen + +#endif // EIGEN_INVERSE_H diff --git a/Eigen/src/Core/Map.h b/Eigen/src/Core/Map.h index ced1b76ba..7dfdc3d59 100644 --- a/Eigen/src/Core/Map.h +++ b/Eigen/src/Core/Map.h @@ -79,10 +79,11 @@ struct traits<Map<PlainObjectType, MapOptions, StrideType> > OuterStrideAtCompileTime = StrideType::OuterStrideAtCompileTime == 0 ? int(PlainObjectType::OuterStrideAtCompileTime) : int(StrideType::OuterStrideAtCompileTime), + IsAligned = bool(EIGEN_ALIGN) && ((int(MapOptions)&Aligned)==Aligned), +#ifndef EIGEN_TEST_EVALUATORS HasNoInnerStride = InnerStrideAtCompileTime == 1, HasNoOuterStride = StrideType::OuterStrideAtCompileTime == 0, HasNoStride = HasNoInnerStride && HasNoOuterStride, - IsAligned = bool(EIGEN_ALIGN) && ((int(MapOptions)&Aligned)==Aligned), IsDynamicSize = PlainObjectType::SizeAtCompileTime==Dynamic, KeepsPacketAccess = bool(HasNoInnerStride) && ( bool(IsDynamicSize) @@ -95,6 +96,10 @@ struct traits<Map<PlainObjectType, MapOptions, StrideType> > ? int(Flags1) : int(Flags1 & ~LinearAccessBit), Flags3 = is_lvalue<PlainObjectType>::value ? int(Flags2) : (int(Flags2) & ~LvalueBit), Flags = KeepsPacketAccess ? int(Flags3) : (int(Flags3) & ~PacketAccessBit) +#else + Flags0 = TraitsBase::Flags & (~NestByRefBit), + Flags = is_lvalue<PlainObjectType>::value ? int(Flags0) : (int(Flags0) & ~LvalueBit) +#endif }; private: enum { Options }; // Expressions don't have Options diff --git a/Eigen/src/Core/MapBase.h b/Eigen/src/Core/MapBase.h index e8ecb175b..927a59c50 100644 --- a/Eigen/src/Core/MapBase.h +++ b/Eigen/src/Core/MapBase.h @@ -161,11 +161,15 @@ template<typename Derived> class MapBase<Derived, ReadOnlyAccessors> EIGEN_DEVICE_FUNC void checkSanity() const { +#ifndef EIGEN_TEST_EVALUATORS + // moved to evaluator EIGEN_STATIC_ASSERT(EIGEN_IMPLIES(internal::traits<Derived>::Flags&PacketAccessBit, internal::inner_stride_at_compile_time<Derived>::ret==1), PACKET_ACCESS_REQUIRES_TO_HAVE_INNER_STRIDE_FIXED_TO_1); - eigen_assert(EIGEN_IMPLIES(internal::traits<Derived>::Flags&AlignedBit, (size_t(m_data) % EIGEN_ALIGN_BYTES) == 0) - && "data is not aligned"); + eigen_assert(EIGEN_IMPLIES(internal::traits<Derived>::Flags&AlignedBit, (size_t(m_data) % EIGEN_ALIGN_BYTES) == 0) && "data is not aligned"); +#else + eigen_assert(EIGEN_IMPLIES(internal::traits<Derived>::IsAligned, (size_t(m_data) % EIGEN_ALIGN_BYTES) == 0) && "data is not aligned"); +#endif } PointerType m_data; diff --git a/Eigen/src/Core/Matrix.h b/Eigen/src/Core/Matrix.h index 8c95ee3ca..1daaabb07 100644 --- a/Eigen/src/Core/Matrix.h +++ b/Eigen/src/Core/Matrix.h @@ -115,7 +115,12 @@ struct traits<Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols> > MaxRowsAtCompileTime = _MaxRows, MaxColsAtCompileTime = _MaxCols, Flags = compute_matrix_flags<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols>::ret, +#ifndef EIGEN_TEST_EVALUATORS CoeffReadCost = NumTraits<Scalar>::ReadCost, +#else + // FIXME, the following flag in only used to define NeedsToAlign in PlainObjectBase + EvaluatorFlags = compute_matrix_evaluator_flags<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols>::ret, +#endif Options = _Options, InnerStrideAtCompileTime = 1, OuterStrideAtCompileTime = (Options&RowMajor) ? ColsAtCompileTime : RowsAtCompileTime diff --git a/Eigen/src/Core/MatrixBase.h b/Eigen/src/Core/MatrixBase.h index f5987d194..c2011d462 100644 --- a/Eigen/src/Core/MatrixBase.h +++ b/Eigen/src/Core/MatrixBase.h @@ -66,8 +66,10 @@ template<typename Derived> class MatrixBase using Base::MaxSizeAtCompileTime; using Base::IsVectorAtCompileTime; using Base::Flags; +#ifndef EIGEN_TEST_EVALUATORS using Base::CoeffReadCost; - +#endif + using Base::derived; using Base::const_cast_derived; using Base::rows; @@ -199,7 +201,11 @@ template<typename Derived> class MatrixBase template<typename OtherDerived> EIGEN_DEVICE_FUNC +#ifdef EIGEN_TEST_EVALUATORS + const Product<Derived,OtherDerived,LazyProduct> +#else const typename LazyProductReturnType<Derived,OtherDerived>::Type +#endif lazyProduct(const MatrixBase<OtherDerived> &other) const; template<typename OtherDerived> @@ -211,10 +217,17 @@ template<typename Derived> class MatrixBase template<typename OtherDerived> void applyOnTheRight(const EigenBase<OtherDerived>& other); +#ifndef EIGEN_TEST_EVALUATORS template<typename DiagonalDerived> EIGEN_DEVICE_FUNC const DiagonalProduct<Derived, DiagonalDerived, OnTheRight> operator*(const DiagonalBase<DiagonalDerived> &diagonal) const; +#else // EIGEN_TEST_EVALUATORS + template<typename DiagonalDerived> + EIGEN_DEVICE_FUNC + const Product<Derived, DiagonalDerived, LazyProduct> + operator*(const DiagonalBase<DiagonalDerived> &diagonal) const; +#endif // EIGEN_TEST_EVALUATORS template<typename OtherDerived> EIGEN_DEVICE_FUNC @@ -333,10 +346,19 @@ template<typename Derived> class MatrixBase NoAlias<Derived,Eigen::MatrixBase > noalias(); +#ifndef EIGEN_TEST_EVALUATORS inline const ForceAlignedAccess<Derived> forceAlignedAccess() const; inline ForceAlignedAccess<Derived> forceAlignedAccess(); template<bool Enable> inline typename internal::add_const_on_value_type<typename internal::conditional<Enable,ForceAlignedAccess<Derived>,Derived&>::type>::type forceAlignedAccessIf() const; template<bool Enable> inline typename internal::conditional<Enable,ForceAlignedAccess<Derived>,Derived&>::type forceAlignedAccessIf(); +#else + // TODO forceAlignedAccess is temporarly disabled + // Need to find a nicer workaround. + inline const Derived& forceAlignedAccess() const { return derived(); } + inline Derived& forceAlignedAccess() { return derived(); } + template<bool Enable> inline const Derived& forceAlignedAccessIf() const { return derived(); } + template<bool Enable> inline Derived& forceAlignedAccessIf() { return derived(); } +#endif Scalar trace() const; @@ -359,8 +381,13 @@ template<typename Derived> class MatrixBase const PartialPivLU<PlainObject> lu() const; + #ifdef EIGEN_TEST_EVALUATORS + EIGEN_DEVICE_FUNC + const Inverse<Derived> inverse() const; + #else EIGEN_DEVICE_FUNC const internal::inverse_impl<Derived> inverse() const; + #endif template<typename ResultType> void computeInverseAndDetWithCheck( ResultType& inverse, diff --git a/Eigen/src/Core/NoAlias.h b/Eigen/src/Core/NoAlias.h index 0a1c32743..fe6dded60 100644 --- a/Eigen/src/Core/NoAlias.h +++ b/Eigen/src/Core/NoAlias.h @@ -30,9 +30,37 @@ namespace Eigen { template<typename ExpressionType, template <typename> class StorageBase> class NoAlias { - typedef typename ExpressionType::Scalar Scalar; public: + typedef typename ExpressionType::Scalar Scalar; + NoAlias(ExpressionType& expression) : m_expression(expression) {} + +#ifdef EIGEN_TEST_EVALUATORS + template<typename OtherDerived> + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE ExpressionType& operator=(const StorageBase<OtherDerived>& other) + { + call_assignment_no_alias(m_expression, other.derived(), internal::assign_op<Scalar>()); + return m_expression; + } + + template<typename OtherDerived> + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE ExpressionType& operator+=(const StorageBase<OtherDerived>& other) + { + call_assignment_no_alias(m_expression, other.derived(), internal::add_assign_op<Scalar>()); + return m_expression; + } + + template<typename OtherDerived> + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE ExpressionType& operator-=(const StorageBase<OtherDerived>& other) + { + call_assignment_no_alias(m_expression, other.derived(), internal::sub_assign_op<Scalar>()); + return m_expression; + } + +#else /** Behaves like MatrixBase::lazyAssign(other) * \sa MatrixBase::lazyAssign() */ @@ -92,6 +120,8 @@ class NoAlias { return m_expression = func; } #endif +#endif + EIGEN_DEVICE_FUNC ExpressionType& expression() const { diff --git a/Eigen/src/Core/PermutationMatrix.h b/Eigen/src/Core/PermutationMatrix.h index 8aa4c8bc5..31e0697a1 100644 --- a/Eigen/src/Core/PermutationMatrix.h +++ b/Eigen/src/Core/PermutationMatrix.h @@ -13,7 +13,8 @@ namespace Eigen { -template<int RowCol,typename IndicesType,typename MatrixType, typename StorageKind> class PermutedImpl; +// TODO: this does not seems to be needed at all: +// template<int RowCol,typename IndicesType,typename MatrixType, typename StorageKind> class PermutedImpl; /** \class PermutationBase * \ingroup Core_Module @@ -60,7 +61,9 @@ class PermutationBase : public EigenBase<Derived> typedef typename Traits::IndicesType IndicesType; enum { Flags = Traits::Flags, +#ifndef EIGEN_TEST_EVALUATORS CoeffReadCost = Traits::CoeffReadCost, +#endif RowsAtCompileTime = Traits::RowsAtCompileTime, ColsAtCompileTime = Traits::ColsAtCompileTime, MaxRowsAtCompileTime = Traits::MaxRowsAtCompileTime, @@ -274,6 +277,7 @@ template<int SizeAtCompileTime, int MaxSizeAtCompileTime, typename _StorageIndex struct traits<PermutationMatrix<SizeAtCompileTime, MaxSizeAtCompileTime, _StorageIndexType> > : traits<Matrix<_StorageIndexType,SizeAtCompileTime,SizeAtCompileTime,0,MaxSizeAtCompileTime,MaxSizeAtCompileTime> > { + typedef PermutationStorage StorageKind; typedef Matrix<_StorageIndexType, SizeAtCompileTime, 1, 0, MaxSizeAtCompileTime, 1> IndicesType; typedef typename IndicesType::Index Index; typedef _StorageIndexType StorageIndexType; @@ -287,6 +291,10 @@ class PermutationMatrix : public PermutationBase<PermutationMatrix<SizeAtCompile typedef internal::traits<PermutationMatrix> Traits; public: +#ifdef EIGEN_TEST_EVALUATORS + typedef const PermutationMatrix& Nested; +#endif + #ifndef EIGEN_PARSED_BY_DOXYGEN typedef typename Traits::IndicesType IndicesType; typedef typename Traits::StorageIndexType StorageIndexType; @@ -391,6 +399,7 @@ template<int SizeAtCompileTime, int MaxSizeAtCompileTime, typename _StorageIndex struct traits<Map<PermutationMatrix<SizeAtCompileTime, MaxSizeAtCompileTime, _StorageIndexType>,_PacketAccess> > : traits<Matrix<_StorageIndexType,SizeAtCompileTime,SizeAtCompileTime,0,MaxSizeAtCompileTime,MaxSizeAtCompileTime> > { + typedef PermutationStorage StorageKind; typedef Map<const Matrix<_StorageIndexType, SizeAtCompileTime, 1, 0, MaxSizeAtCompileTime, 1>, _PacketAccess> IndicesType; typedef typename IndicesType::Index Index; typedef _StorageIndexType StorageIndexType; @@ -462,8 +471,6 @@ class Map<PermutationMatrix<SizeAtCompileTime, MaxSizeAtCompileTime, _StorageInd * \sa class PermutationBase, class PermutationMatrix */ -struct PermutationStorage {}; - template<typename _IndicesType> class TranspositionsWrapper; namespace internal { template<typename _IndicesType> @@ -477,10 +484,18 @@ struct traits<PermutationWrapper<_IndicesType> > enum { RowsAtCompileTime = _IndicesType::SizeAtCompileTime, ColsAtCompileTime = _IndicesType::SizeAtCompileTime, - MaxRowsAtCompileTime = IndicesType::MaxRowsAtCompileTime, +#ifdef EIGEN_TEST_EVALUATORS + MaxRowsAtCompileTime = IndicesType::MaxSizeAtCompileTime, + MaxColsAtCompileTime = IndicesType::MaxSizeAtCompileTime, +#else + MaxRowsAtCompileTime = IndicesType::MaxRowsAtCompileTime, // is this a bug in Eigen 2.2 ? MaxColsAtCompileTime = IndicesType::MaxColsAtCompileTime, - Flags = 0, +#endif + Flags = 0 +#ifndef EIGEN_TEST_EVALUATORS + , CoeffReadCost = _IndicesType::CoeffReadCost +#endif }; }; } @@ -509,6 +524,37 @@ class PermutationWrapper : public PermutationBase<PermutationWrapper<_IndicesTyp typename IndicesType::Nested m_indices; }; +#ifdef EIGEN_TEST_EVALUATORS + +// TODO: Do we need to define these operator* functions? Would it be better to have them inherited +// from MatrixBase? + +/** \returns the matrix with the permutation applied to the columns. + */ +template<typename MatrixDerived, typename PermutationDerived> +EIGEN_DEVICE_FUNC +const Product<MatrixDerived, PermutationDerived, DefaultProduct> +operator*(const MatrixBase<MatrixDerived> &matrix, + const PermutationBase<PermutationDerived>& permutation) +{ + return Product<MatrixDerived, PermutationDerived, DefaultProduct> + (matrix.derived(), permutation.derived()); +} + +/** \returns the matrix with the permutation applied to the rows. + */ +template<typename PermutationDerived, typename MatrixDerived> +EIGEN_DEVICE_FUNC +const Product<PermutationDerived, MatrixDerived, DefaultProduct> +operator*(const PermutationBase<PermutationDerived> &permutation, + const MatrixBase<MatrixDerived>& matrix) +{ + return Product<PermutationDerived, MatrixDerived, DefaultProduct> + (permutation.derived(), matrix.derived()); +} + +#else // EIGEN_TEST_EVALUATORS + /** \returns the matrix with the permutation applied to the columns. */ template<typename Derived, typename PermutationDerived> @@ -534,10 +580,13 @@ operator*(const PermutationBase<PermutationDerived> &permutation, (permutation.derived(), matrix.derived()); } +#endif // EIGEN_TEST_EVALUATORS + namespace internal { template<typename PermutationType, typename MatrixType, int Side, bool Transposed> struct traits<permut_matrix_product_retval<PermutationType, MatrixType, Side, Transposed> > + : traits<typename MatrixType::PlainObject> { typedef typename MatrixType::PlainObject ReturnType; }; @@ -617,6 +666,8 @@ struct traits<Transpose<PermutationBase<Derived> > > } // end namespace internal +// TODO: the specificties should be handled by the evaluator, +// at the very least we should only specialize TransposeImpl template<typename Derived> class Transpose<PermutationBase<Derived> > : public EigenBase<Transpose<PermutationBase<Derived> > > @@ -631,7 +682,9 @@ class Transpose<PermutationBase<Derived> > typedef typename Derived::DenseMatrixType DenseMatrixType; enum { Flags = Traits::Flags, +#ifndef EIGEN_TEST_EVALUATORS CoeffReadCost = Traits::CoeffReadCost, +#endif RowsAtCompileTime = Traits::RowsAtCompileTime, ColsAtCompileTime = Traits::ColsAtCompileTime, MaxRowsAtCompileTime = Traits::MaxRowsAtCompileTime, @@ -660,6 +713,28 @@ class Transpose<PermutationBase<Derived> > DenseMatrixType toDenseMatrix() const { return *this; } +#ifdef EIGEN_TEST_EVALUATORS + + /** \returns the matrix with the inverse permutation applied to the columns. + */ + template<typename OtherDerived> friend + const Product<OtherDerived, Transpose, DefaultProduct> + operator*(const MatrixBase<OtherDerived>& matrix, const Transpose& trPerm) + { + return Product<OtherDerived, Transpose, DefaultProduct>(matrix.derived(), trPerm.derived()); + } + + /** \returns the matrix with the inverse permutation applied to the rows. + */ + template<typename OtherDerived> + const Product<Transpose, OtherDerived, DefaultProduct> + operator*(const MatrixBase<OtherDerived>& matrix) const + { + return Product<Transpose, OtherDerived, DefaultProduct>(*this, matrix.derived()); + } + +#else // EIGEN_TEST_EVALUATORS + /** \returns the matrix with the inverse permutation applied to the columns. */ template<typename OtherDerived> friend @@ -678,6 +753,8 @@ class Transpose<PermutationBase<Derived> > return internal::permut_matrix_product_retval<PermutationType, OtherDerived, OnTheLeft, true>(m_permutation, matrix.derived()); } +#endif // EIGEN_TEST_EVALUATORS + const PermutationType& nestedPermutation() const { return m_permutation; } protected: @@ -690,6 +767,40 @@ const PermutationWrapper<const Derived> MatrixBase<Derived>::asPermutation() con return derived(); } +#ifdef EIGEN_TEST_EVALUATORS +namespace internal { + +// TODO currently a permutation matrix expression has the form PermutationMatrix or PermutationWrapper +// or their transpose; in the future shape should be defined by the expression traits +template<int SizeAtCompileTime, int MaxSizeAtCompileTime, typename IndexType> +struct evaluator_traits<PermutationMatrix<SizeAtCompileTime, MaxSizeAtCompileTime, IndexType> > +{ + typedef typename storage_kind_to_evaluator_kind<Dense>::Kind Kind; + typedef PermutationShape Shape; + static const int AssumeAliasing = 0; +}; + +template<typename IndicesType> +struct evaluator_traits<PermutationWrapper<IndicesType> > +{ + typedef typename storage_kind_to_evaluator_kind<Dense>::Kind Kind; + typedef PermutationShape Shape; + static const int AssumeAliasing = 0; +}; + +template<typename Derived> +struct evaluator_traits<Transpose<PermutationBase<Derived> > > +{ + typedef typename storage_kind_to_evaluator_kind<Dense>::Kind Kind; + typedef PermutationShape Shape; + static const int AssumeAliasing = 0; +}; + +template<> struct AssignmentKind<DenseShape,PermutationShape> { typedef EigenBase2EigenBase Kind; }; + +} // end namespace internal +#endif // EIGEN_TEST_EVALUATORS + } // end namespace Eigen #endif // EIGEN_PERMUTATIONMATRIX_H diff --git a/Eigen/src/Core/PlainObjectBase.h b/Eigen/src/Core/PlainObjectBase.h index 69f34bd3e..3637b6256 100644 --- a/Eigen/src/Core/PlainObjectBase.h +++ b/Eigen/src/Core/PlainObjectBase.h @@ -128,7 +128,11 @@ class PlainObjectBase : public internal::dense_xpr_base<Derived>::type DenseStorage<Scalar, Base::MaxSizeAtCompileTime, Base::RowsAtCompileTime, Base::ColsAtCompileTime, Options> m_storage; public: +#ifndef EIGEN_TEST_EVALUATORS enum { NeedsToAlign = SizeAtCompileTime != Dynamic && (internal::traits<Derived>::Flags & AlignedBit) != 0 }; +#else + enum { NeedsToAlign = SizeAtCompileTime != Dynamic && (internal::traits<Derived>::EvaluatorFlags & AlignedBit) != 0 }; +#endif EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF(NeedsToAlign) EIGEN_DEVICE_FUNC @@ -639,6 +643,18 @@ class PlainObjectBase : public internal::dense_xpr_base<Derived>::type * * \internal */ +#ifdef EIGEN_TEST_EVALUATORS + // aliasing is dealt once in internall::call_assignment + // so at this stage we have to assume aliasing... and resising has to be done later. + template<typename OtherDerived> + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE Derived& _set(const DenseBase<OtherDerived>& other) + { + internal::call_assignment(this->derived(), other.derived()); + return this->derived(); + return this->derived(); + } +#else template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& _set(const DenseBase<OtherDerived>& other) @@ -654,7 +670,7 @@ class PlainObjectBase : public internal::dense_xpr_base<Derived>::type template<typename OtherDerived> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void _set_selector(const OtherDerived& other, const internal::false_type&) { _set_noalias(other); } - +#endif /** \internal Like _set() but additionally makes the assumption that no aliasing effect can happen (which * is the case when creating a new matrix) so one can enforce lazy evaluation. * @@ -669,7 +685,12 @@ class PlainObjectBase : public internal::dense_xpr_base<Derived>::type //_resize_to_match(other); // the 'false' below means to enforce lazy evaluation. We don't use lazyAssign() because // it wouldn't allow to copy a row-vector into a column-vector. +#ifdef EIGEN_TEST_EVALUATORS + internal::call_assignment_no_alias(this->derived(), other.derived(), internal::assign_op<Scalar>()); + return this->derived(); +#else return internal::assign_selector<Derived,OtherDerived,false>::run(this->derived(), other.derived()); +#endif } template<typename T0, typename T1> diff --git a/Eigen/src/Core/Product.h b/Eigen/src/Core/Product.h index 5d3789be7..0cf20f2e2 100644 --- a/Eigen/src/Core/Product.h +++ b/Eigen/src/Core/Product.h @@ -12,8 +12,7 @@ namespace Eigen { -template<typename Lhs, typename Rhs> class Product; -template<typename Lhs, typename Rhs, typename StorageKind> class ProductImpl; +template<typename Lhs, typename Rhs, int Option, typename StorageKind> class ProductImpl; /** \class Product * \ingroup Core_Module @@ -24,38 +23,98 @@ template<typename Lhs, typename Rhs, typename StorageKind> class ProductImpl; * \param Rhs the type of the right-hand side expression * * This class represents an expression of the product of two arbitrary matrices. + * + * The other template parameters are: + * \tparam Option can be DefaultProduct or LazyProduct * */ -// Use ProductReturnType to get correct traits, in particular vectorization flags + namespace internal { -template<typename Lhs, typename Rhs> -struct traits<Product<Lhs, Rhs> > - : traits<typename ProductReturnType<Lhs, Rhs>::Type> -{ - // We want A+B*C to be of type Product<Matrix, Sum> and not Product<Matrix, Matrix> - // TODO: This flag should eventually go in a separate evaluator traits class + +// Determine the scalar of Product<Lhs, Rhs>. This is normally the same as Lhs::Scalar times +// Rhs::Scalar, but product with permutation matrices inherit the scalar of the other factor. +template<typename Lhs, typename Rhs, typename LhsShape = typename evaluator_traits<Lhs>::Shape, + typename RhsShape = typename evaluator_traits<Rhs>::Shape > +struct product_result_scalar +{ + typedef typename scalar_product_traits<typename Lhs::Scalar, typename Rhs::Scalar>::ReturnType Scalar; +}; + +template<typename Lhs, typename Rhs, typename RhsShape> +struct product_result_scalar<Lhs, Rhs, PermutationShape, RhsShape> +{ + typedef typename Rhs::Scalar Scalar; +}; + +template<typename Lhs, typename Rhs, typename LhsShape> + struct product_result_scalar<Lhs, Rhs, LhsShape, PermutationShape> +{ + typedef typename Lhs::Scalar Scalar; +}; + +template<typename Lhs, typename Rhs, int Option> +struct traits<Product<Lhs, Rhs, Option> > +{ + typedef typename remove_all<Lhs>::type LhsCleaned; + typedef typename remove_all<Rhs>::type RhsCleaned; + typedef traits<LhsCleaned> LhsTraits; + typedef traits<RhsCleaned> RhsTraits; + + typedef MatrixXpr XprKind; + + typedef typename product_result_scalar<LhsCleaned,RhsCleaned>::Scalar Scalar; + typedef typename product_promote_storage_type<typename LhsTraits::StorageKind, + typename RhsTraits::StorageKind, + internal::product_type<Lhs,Rhs>::ret>::ret StorageKind; + typedef typename promote_index_type<typename LhsTraits::Index, + typename RhsTraits::Index>::type Index; + enum { - Flags = traits<typename ProductReturnType<Lhs, Rhs>::Type>::Flags & ~(EvalBeforeNestingBit | DirectAccessBit) + RowsAtCompileTime = LhsTraits::RowsAtCompileTime, + ColsAtCompileTime = RhsTraits::ColsAtCompileTime, + MaxRowsAtCompileTime = LhsTraits::MaxRowsAtCompileTime, + MaxColsAtCompileTime = RhsTraits::MaxColsAtCompileTime, + + // FIXME: only needed by GeneralMatrixMatrixTriangular + InnerSize = EIGEN_SIZE_MIN_PREFER_FIXED(LhsTraits::ColsAtCompileTime, RhsTraits::RowsAtCompileTime), + +#ifndef EIGEN_TEST_EVALUATORS + // dummy, for evaluators unit test only + CoeffReadCost = Dynamic, +#endif + + // The storage order is somewhat arbitrary here. The correct one will be determined through the evaluator. + Flags = ( MaxRowsAtCompileTime==1 + || ((LhsTraits::Flags&NoPreferredStorageOrderBit) && (RhsTraits::Flags&RowMajorBit)) + || ((RhsTraits::Flags&NoPreferredStorageOrderBit) && (LhsTraits::Flags&RowMajorBit)) ) + ? RowMajorBit : (MaxColsAtCompileTime==1 ? 0 : NoPreferredStorageOrderBit) }; }; + } // end namespace internal -template<typename Lhs, typename Rhs> -class Product : public ProductImpl<Lhs,Rhs,typename internal::promote_storage_type<typename internal::traits<Lhs>::StorageKind, - typename internal::traits<Rhs>::StorageKind>::ret> +template<typename _Lhs, typename _Rhs, int Option> +class Product : public ProductImpl<_Lhs,_Rhs,Option, + typename internal::product_promote_storage_type<typename internal::traits<_Lhs>::StorageKind, + typename internal::traits<_Rhs>::StorageKind, + internal::product_type<_Lhs,_Rhs>::ret>::ret> { public: + typedef _Lhs Lhs; + typedef _Rhs Rhs; + typedef typename ProductImpl< - Lhs, Rhs, - typename internal::promote_storage_type<typename Lhs::StorageKind, - typename Rhs::StorageKind>::ret>::Base Base; + Lhs, Rhs, Option, + typename internal::product_promote_storage_type<typename internal::traits<Lhs>::StorageKind, + typename internal::traits<Rhs>::StorageKind, + internal::product_type<Lhs,Rhs>::ret>::ret>::Base Base; EIGEN_GENERIC_PUBLIC_INTERFACE(Product) - typedef typename Lhs::Nested LhsNested; - typedef typename Rhs::Nested RhsNested; + typedef typename internal::nested<Lhs>::type LhsNested; + typedef typename internal::nested<Rhs>::type RhsNested; typedef typename internal::remove_all<LhsNested>::type LhsNestedCleaned; typedef typename internal::remove_all<RhsNested>::type RhsNestedCleaned; @@ -78,14 +137,79 @@ class Product : public ProductImpl<Lhs,Rhs,typename internal::promote_storage_ty RhsNested m_rhs; }; -template<typename Lhs, typename Rhs> -class ProductImpl<Lhs,Rhs,Dense> : public internal::dense_xpr_base<Product<Lhs,Rhs> >::type +namespace internal { + +template<typename Lhs, typename Rhs, int Option, int ProductTag = internal::product_type<Lhs,Rhs>::ret> +class dense_product_base + : public internal::dense_xpr_base<Product<Lhs,Rhs,Option> >::type +{}; + +/** Convertion to scalar for inner-products */ +template<typename Lhs, typename Rhs, int Option> +class dense_product_base<Lhs, Rhs, Option, InnerProduct> + : public internal::dense_xpr_base<Product<Lhs,Rhs,Option> >::type +{ + typedef Product<Lhs,Rhs,Option> ProductXpr; + typedef typename internal::dense_xpr_base<ProductXpr>::type Base; +public: + using Base::derived; + typedef typename Base::Scalar Scalar; + typedef typename Base::Index Index; + + operator const Scalar() const + { + return typename internal::evaluator<ProductXpr>::type(derived()).coeff(0,0); + } +}; + +} // namespace internal + +#ifdef EIGEN_TEST_EVALUATORS +// Generic API dispatcher +template<typename Lhs, typename Rhs, int Option, typename StorageKind> +class ProductImpl : public internal::generic_xpr_base<Product<Lhs,Rhs,Option>, MatrixXpr, StorageKind>::type { - typedef Product<Lhs, Rhs> Derived; public: + typedef typename internal::generic_xpr_base<Product<Lhs,Rhs,Option>, MatrixXpr, StorageKind>::type Base; +}; +#endif - typedef typename internal::dense_xpr_base<Product<Lhs, Rhs> >::type Base; +template<typename Lhs, typename Rhs, int Option> +class ProductImpl<Lhs,Rhs,Option,Dense> + : public internal::dense_product_base<Lhs,Rhs,Option> +{ + typedef Product<Lhs, Rhs, Option> Derived; + + public: + + typedef typename internal::dense_product_base<Lhs, Rhs, Option> Base; EIGEN_DENSE_PUBLIC_INTERFACE(Derived) + protected: + enum { + IsOneByOne = (RowsAtCompileTime == 1 || RowsAtCompileTime == Dynamic) && + (ColsAtCompileTime == 1 || ColsAtCompileTime == Dynamic), + EnableCoeff = IsOneByOne || Option==LazyProduct + }; + + public: + + Scalar coeff(Index row, Index col) const + { + EIGEN_STATIC_ASSERT(EnableCoeff, THIS_METHOD_IS_ONLY_FOR_INNER_OR_LAZY_PRODUCTS); + eigen_assert( (Option==LazyProduct) || (this->rows() == 1 && this->cols() == 1) ); + + return typename internal::evaluator<Derived>::type(derived()).coeff(row,col); + } + + Scalar coeff(Index i) const + { + EIGEN_STATIC_ASSERT(EnableCoeff, THIS_METHOD_IS_ONLY_FOR_INNER_OR_LAZY_PRODUCTS); + eigen_assert( (Option==LazyProduct) || (this->rows() == 1 && this->cols() == 1) ); + + return typename internal::evaluator<Derived>::type(derived()).coeff(i); + } + + }; /*************************************************************************** @@ -102,6 +226,15 @@ prod(const Lhs& lhs, const Rhs& rhs) return Product<Lhs,Rhs>(lhs,rhs); } +/** \internal used to test the evaluator only + */ +template<typename Lhs,typename Rhs> +const Product<Lhs,Rhs,LazyProduct> +lazyprod(const Lhs& lhs, const Rhs& rhs) +{ + return Product<Lhs,Rhs,LazyProduct>(lhs,rhs); +} + } // end namespace Eigen #endif // EIGEN_PRODUCT_H diff --git a/Eigen/src/Core/ProductBase.h b/Eigen/src/Core/ProductBase.h index 483914a9b..4b1fe356b 100644 --- a/Eigen/src/Core/ProductBase.h +++ b/Eigen/src/Core/ProductBase.h @@ -12,6 +12,8 @@ namespace Eigen { +#ifndef EIGEN_TEST_EVALUATORS + /** \class ProductBase * \ingroup Core_Module * @@ -25,8 +27,9 @@ struct traits<ProductBase<Derived,_Lhs,_Rhs> > typedef typename remove_all<_Lhs>::type Lhs; typedef typename remove_all<_Rhs>::type Rhs; typedef typename scalar_product_traits<typename Lhs::Scalar, typename Rhs::Scalar>::ReturnType Scalar; - typedef typename promote_storage_type<typename traits<Lhs>::StorageKind, - typename traits<Rhs>::StorageKind>::ret StorageKind; + typedef typename product_promote_storage_type<typename traits<Lhs>::StorageKind, + typename traits<Rhs>::StorageKind, + 0>::ret StorageKind; typedef typename promote_index_type<typename traits<Lhs>::Index, typename traits<Rhs>::Index>::type Index; enum { @@ -259,6 +262,8 @@ class ScaledProduct Scalar m_alpha; }; +#endif // EIGEN_TEST_EVALUATORS + /** \internal * Overloaded to perform an efficient C = (A*B).lazy() */ template<typename Derived> diff --git a/Eigen/src/Core/ProductEvaluators.h b/Eigen/src/Core/ProductEvaluators.h index 855914f2e..8a63384a7 100644 --- a/Eigen/src/Core/ProductEvaluators.h +++ b/Eigen/src/Core/ProductEvaluators.h @@ -16,95 +16,344 @@ namespace Eigen { namespace internal { + +/** \internal + * Evaluator of a product expression. + * Since products require special treatments to handle all possible cases, + * we simply deffer the evaluation logic to a product_evaluator class + * which offers more partial specialization possibilities. + * + * \sa class product_evaluator + */ +template<typename Lhs, typename Rhs, int Options> +struct evaluator<Product<Lhs, Rhs, Options> > + : public product_evaluator<Product<Lhs, Rhs, Options> > +{ + typedef Product<Lhs, Rhs, Options> XprType; + typedef product_evaluator<XprType> Base; + + typedef evaluator type; + typedef evaluator nestedType; -// We can evaluate the product either all at once, like GeneralProduct and its evalTo() function, or -// traverse the matrix coefficient by coefficient, like CoeffBasedProduct. Use the existing logic -// in ProductReturnType to decide. + evaluator(const XprType& xpr) : Base(xpr) {} +}; + +// Catch scalar * ( A * B ) and transform it to (A*scalar) * B +// TODO we should apply that rule only if that's really helpful +template<typename Lhs, typename Rhs, typename Scalar> +struct evaluator<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const Product<Lhs, Rhs, DefaultProduct> > > + : public evaluator<Product<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>,const Lhs>, Rhs, DefaultProduct> > +{ + typedef CwiseUnaryOp<internal::scalar_multiple_op<Scalar>, const Product<Lhs, Rhs, DefaultProduct> > XprType; + typedef evaluator<Product<CwiseUnaryOp<internal::scalar_multiple_op<Scalar>,const Lhs>, Rhs, DefaultProduct> > Base; + + typedef evaluator type; + typedef evaluator nestedType; + + evaluator(const XprType& xpr) + : Base(xpr.functor().m_other * xpr.nestedExpression().lhs() * xpr.nestedExpression().rhs()) + {} +}; -template<typename XprType, typename ProductType> -struct product_evaluator_dispatcher; + +template<typename Lhs, typename Rhs, int DiagIndex> +struct evaluator<Diagonal<const Product<Lhs, Rhs, DefaultProduct>, DiagIndex> > + : public evaluator<Diagonal<const Product<Lhs, Rhs, LazyProduct>, DiagIndex> > +{ + typedef Diagonal<const Product<Lhs, Rhs, DefaultProduct>, DiagIndex> XprType; + typedef evaluator<Diagonal<const Product<Lhs, Rhs, LazyProduct>, DiagIndex> > Base; + + typedef evaluator type; + typedef evaluator nestedType; + + evaluator(const XprType& xpr) + : Base(Diagonal<const Product<Lhs, Rhs, LazyProduct>, DiagIndex>( + Product<Lhs, Rhs, LazyProduct>(xpr.nestedExpression().lhs(), xpr.nestedExpression().rhs()), + xpr.index() )) + {} +}; + + +// Helper class to perform a matrix product with the destination at hand. +// Depending on the sizes of the factors, there are different evaluation strategies +// as controlled by internal::product_type. +template< typename Lhs, typename Rhs, + typename LhsShape = typename evaluator_traits<Lhs>::Shape, + typename RhsShape = typename evaluator_traits<Rhs>::Shape, + int ProductType = internal::product_type<Lhs,Rhs>::value> +struct generic_product_impl; template<typename Lhs, typename Rhs> -struct evaluator_impl<Product<Lhs, Rhs> > - : product_evaluator_dispatcher<Product<Lhs, Rhs>, typename ProductReturnType<Lhs, Rhs>::Type> +struct evaluator_traits<Product<Lhs, Rhs, DefaultProduct> > + : evaluator_traits_base<Product<Lhs, Rhs, DefaultProduct> > { - typedef Product<Lhs, Rhs> XprType; - typedef product_evaluator_dispatcher<XprType, typename ProductReturnType<Lhs, Rhs>::Type> Base; + enum { AssumeAliasing = 1 }; +}; - evaluator_impl(const XprType& xpr) : Base(xpr) - { } +// This is the default evaluator implementation for products: +// It creates a temporary and call generic_product_impl +template<typename Lhs, typename Rhs, int ProductTag, typename LhsShape, typename RhsShape> +struct product_evaluator<Product<Lhs, Rhs, DefaultProduct>, ProductTag, LhsShape, RhsShape, typename traits<Lhs>::Scalar, typename traits<Rhs>::Scalar> + : public evaluator<typename Product<Lhs, Rhs, DefaultProduct>::PlainObject>::type +{ + typedef Product<Lhs, Rhs, DefaultProduct> XprType; +// enum { +// CoeffReadCost = 0 // FIXME why is it needed? (this was already the case before the evaluators, see traits<ProductBase>) +// }; + typedef typename XprType::PlainObject PlainObject; + typedef typename evaluator<PlainObject>::type Base; + + product_evaluator(const XprType& xpr) + : m_result(xpr.rows(), xpr.cols()) + { + ::new (static_cast<Base*>(this)) Base(m_result); + +// FIXME shall we handle nested_eval here? +// typedef typename internal::nested_eval<Lhs,Rhs::ColsAtCompileTime>::type LhsNested; +// typedef typename internal::nested_eval<Rhs,Lhs::RowsAtCompileTime>::type RhsNested; +// typedef typename internal::remove_all<LhsNested>::type LhsNestedCleaned; +// typedef typename internal::remove_all<RhsNested>::type RhsNestedCleaned; +// +// const LhsNested lhs(xpr.lhs()); +// const RhsNested rhs(xpr.rhs()); +// +// generic_product_impl<LhsNestedCleaned, RhsNestedCleaned>::evalTo(m_result, lhs, rhs); + + generic_product_impl<Lhs, Rhs, LhsShape, RhsShape, ProductTag>::evalTo(m_result, xpr.lhs(), xpr.rhs()); + } + +protected: + PlainObject m_result; }; -template<typename XprType, typename ProductType> -struct product_evaluator_traits_dispatcher; +// Dense = Product +template< typename DstXprType, typename Lhs, typename Rhs, typename Scalar> +struct Assignment<DstXprType, Product<Lhs,Rhs,DefaultProduct>, internal::assign_op<Scalar>, Dense2Dense, Scalar> +{ + typedef Product<Lhs,Rhs,DefaultProduct> SrcXprType; + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + { + // FIXME shall we handle nested_eval here? + generic_product_impl<Lhs, Rhs>::evalTo(dst, src.lhs(), src.rhs()); + } +}; -template<typename Lhs, typename Rhs> -struct evaluator_traits<Product<Lhs, Rhs> > - : product_evaluator_traits_dispatcher<Product<Lhs, Rhs>, typename ProductReturnType<Lhs, Rhs>::Type> -{ - static const int AssumeAliasing = 1; +// Dense += Product +template< typename DstXprType, typename Lhs, typename Rhs, typename Scalar> +struct Assignment<DstXprType, Product<Lhs,Rhs,DefaultProduct>, internal::add_assign_op<Scalar>, Dense2Dense, Scalar> +{ + typedef Product<Lhs,Rhs,DefaultProduct> SrcXprType; + static void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op<Scalar> &) + { + // FIXME shall we handle nested_eval here? + generic_product_impl<Lhs, Rhs>::addTo(dst, src.lhs(), src.rhs()); + } }; -// Case 1: Evaluate all at once -// -// We can view the GeneralProduct class as a part of the product evaluator. -// Four sub-cases: InnerProduct, OuterProduct, GemmProduct and GemvProduct. -// InnerProduct is special because GeneralProduct does not have an evalTo() method in this case. +// Dense -= Product +template< typename DstXprType, typename Lhs, typename Rhs, typename Scalar> +struct Assignment<DstXprType, Product<Lhs,Rhs,DefaultProduct>, internal::sub_assign_op<Scalar>, Dense2Dense, Scalar> +{ + typedef Product<Lhs,Rhs,DefaultProduct> SrcXprType; + static void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op<Scalar> &) + { + // FIXME shall we handle nested_eval here? + generic_product_impl<Lhs, Rhs>::subTo(dst, src.lhs(), src.rhs()); + } +}; -template<typename Lhs, typename Rhs> -struct product_evaluator_traits_dispatcher<Product<Lhs, Rhs>, GeneralProduct<Lhs, Rhs, InnerProduct> > + +// Dense ?= scalar * Product +// TODO we should apply that rule if that's really helpful +// for instance, this is not good for inner products +template< typename DstXprType, typename Lhs, typename Rhs, typename AssignFunc, typename Scalar, typename ScalarBis> +struct Assignment<DstXprType, CwiseUnaryOp<internal::scalar_multiple_op<ScalarBis>, + const Product<Lhs,Rhs,DefaultProduct> >, AssignFunc, Dense2Dense, Scalar> { - static const int HasEvalTo = 0; + typedef CwiseUnaryOp<internal::scalar_multiple_op<ScalarBis>, + const Product<Lhs,Rhs,DefaultProduct> > SrcXprType; + static void run(DstXprType &dst, const SrcXprType &src, const AssignFunc& func) + { + // TODO use operator* instead of prod() once we have made enough progress + call_assignment(dst.noalias(), prod(src.functor().m_other * src.nestedExpression().lhs(), src.nestedExpression().rhs()), func); + } }; + template<typename Lhs, typename Rhs> -struct product_evaluator_dispatcher<Product<Lhs, Rhs>, GeneralProduct<Lhs, Rhs, InnerProduct> > - : public evaluator<typename Product<Lhs, Rhs>::PlainObject>::type +struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,InnerProduct> { - typedef Product<Lhs, Rhs> XprType; - typedef typename XprType::PlainObject PlainObject; - typedef typename evaluator<PlainObject>::type evaluator_base; + template<typename Dst> + static inline void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { + dst.coeffRef(0,0) = (lhs.transpose().cwiseProduct(rhs)).sum(); + } + + template<typename Dst> + static inline void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { + dst.coeffRef(0,0) += (lhs.transpose().cwiseProduct(rhs)).sum(); + } + + template<typename Dst> + static void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { dst.coeffRef(0,0) -= (lhs.transpose().cwiseProduct(rhs)).sum(); } +}; + - // TODO: Computation is too early (?) - product_evaluator_dispatcher(const XprType& xpr) : evaluator_base(m_result) +/*********************************************************************** +* Implementation of outer dense * dense vector product +***********************************************************************/ + +// Column major result +template<typename Dst, typename Lhs, typename Rhs, typename Func> +EIGEN_DONT_INLINE void outer_product_selector_run(Dst& dst, const Lhs &lhs, const Rhs &rhs, const Func& func, const false_type&) +{ + typedef typename Dst::Index Index; + // FIXME make sure lhs is sequentially stored + // FIXME not very good if rhs is real and lhs complex while alpha is real too + // FIXME we should probably build an evaluator for dst and rhs + const Index cols = dst.cols(); + for (Index j=0; j<cols; ++j) + func(dst.col(j), rhs.coeff(j) * lhs); +} + +// Row major result +template<typename Dst, typename Lhs, typename Rhs, typename Func> +EIGEN_DONT_INLINE void outer_product_selector_run(Dst& dst, const Lhs &lhs, const Rhs &rhs, const Func& func, const true_type&) { + typedef typename Dst::Index Index; + // FIXME make sure rhs is sequentially stored + // FIXME not very good if lhs is real and rhs complex while alpha is real too + // FIXME we should probably build an evaluator for dst and lhs + const Index rows = dst.rows(); + for (Index i=0; i<rows; ++i) + func(dst.row(i), lhs.coeff(i) * rhs); +} + +template<typename Lhs, typename Rhs> +struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,OuterProduct> +{ + template<typename T> struct IsRowMajor : internal::conditional<(int(T::Flags)&RowMajorBit), internal::true_type, internal::false_type>::type {}; + typedef typename Product<Lhs,Rhs>::Scalar Scalar; + + // TODO it would be nice to be able to exploit our *_assign_op functors for that purpose + struct set { template<typename Dst, typename Src> void operator()(const Dst& dst, const Src& src) const { dst.const_cast_derived() = src; } }; + struct add { template<typename Dst, typename Src> void operator()(const Dst& dst, const Src& src) const { dst.const_cast_derived() += src; } }; + struct sub { template<typename Dst, typename Src> void operator()(const Dst& dst, const Src& src) const { dst.const_cast_derived() -= src; } }; + struct adds { + Scalar m_scale; + adds(const Scalar& s) : m_scale(s) {} + template<typename Dst, typename Src> void operator()(const Dst& dst, const Src& src) const { + dst.const_cast_derived() += m_scale * src; + } + }; + + template<typename Dst> + static inline void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { - m_result.coeffRef(0,0) = (xpr.lhs().transpose().cwiseProduct(xpr.rhs())).sum(); + internal::outer_product_selector_run(dst, lhs, rhs, set(), IsRowMajor<Dst>()); + } + + template<typename Dst> + static inline void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { + internal::outer_product_selector_run(dst, lhs, rhs, add(), IsRowMajor<Dst>()); + } + + template<typename Dst> + static inline void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { + internal::outer_product_selector_run(dst, lhs, rhs, sub(), IsRowMajor<Dst>()); + } + + template<typename Dst> + static inline void scaleAndAddTo(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) + { + internal::outer_product_selector_run(dst, lhs, rhs, adds(alpha), IsRowMajor<Dst>()); } -protected: - PlainObject m_result; }; -// For the other three subcases, simply call the evalTo() method of GeneralProduct -// TODO: GeneralProduct should take evaluators, not expression objects. -template<typename Lhs, typename Rhs, int ProductType> -struct product_evaluator_traits_dispatcher<Product<Lhs, Rhs>, GeneralProduct<Lhs, Rhs, ProductType> > +// This base class provides default implementations for evalTo, addTo, subTo, in terms of scaleAndAddTo +template<typename Lhs, typename Rhs, typename Derived> +struct generic_product_impl_base { - static const int HasEvalTo = 1; + typedef typename Product<Lhs,Rhs>::Scalar Scalar; + + template<typename Dst> + static void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { dst.setZero(); scaleAndAddTo(dst, lhs, rhs, Scalar(1)); } + + template<typename Dst> + static void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { scaleAndAddTo(dst,lhs, rhs, Scalar(1)); } + + template<typename Dst> + static void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { scaleAndAddTo(dst, lhs, rhs, Scalar(-1)); } + + template<typename Dst> + static void scaleAndAddTo(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) + { Derived::scaleAndAddTo(dst,lhs,rhs,alpha); } + }; -template<typename Lhs, typename Rhs, int ProductType> -struct product_evaluator_dispatcher<Product<Lhs, Rhs>, GeneralProduct<Lhs, Rhs, ProductType> > +template<typename Lhs, typename Rhs> +struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,GemvProduct> + : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,GemvProduct> > { - typedef Product<Lhs, Rhs> XprType; - typedef typename XprType::PlainObject PlainObject; - typedef typename evaluator<PlainObject>::type evaluator_base; + typedef typename Product<Lhs,Rhs>::Scalar Scalar; + enum { Side = Lhs::IsVectorAtCompileTime ? OnTheLeft : OnTheRight }; + typedef typename internal::conditional<int(Side)==OnTheRight,Lhs,Rhs>::type MatrixType; + + template<typename Dest> + static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) + { + internal::gemv_dense_sense_selector<Side, + (int(MatrixType::Flags)&RowMajorBit) ? RowMajor : ColMajor, + bool(internal::blas_traits<MatrixType>::HasUsableDirectAccess) + >::run(lhs, rhs, dst, alpha); + } +}; + +template<typename Lhs, typename Rhs> +struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,CoeffBasedProductMode> +{ + typedef typename Product<Lhs,Rhs>::Scalar Scalar; - product_evaluator_dispatcher(const XprType& xpr) : m_xpr(xpr) - { } + template<typename Dst> + static inline void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { + // TODO: use the following instead of calling call_assignment, same for the other methods + // dst = lazyprod(lhs,rhs); + call_assignment(dst, lazyprod(lhs,rhs), internal::assign_op<Scalar>()); + } + + template<typename Dst> + static inline void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { + // dst += lazyprod(lhs,rhs); + call_assignment(dst, lazyprod(lhs,rhs), internal::add_assign_op<Scalar>()); + } - template<typename DstEvaluatorType, typename DstXprType> - void evalTo(DstEvaluatorType /* not used */, DstXprType& dst) const + template<typename Dst> + static inline void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) { - dst.resize(m_xpr.rows(), m_xpr.cols()); - GeneralProduct<Lhs, Rhs, ProductType>(m_xpr.lhs(), m_xpr.rhs()).evalTo(dst); + // dst -= lazyprod(lhs,rhs); + call_assignment(dst, lazyprod(lhs,rhs), internal::sub_assign_op<Scalar>()); } -protected: - const XprType& m_xpr; +// template<typename Dst> +// static inline void scaleAndAddTo(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) +// { dst += alpha * lazyprod(lhs,rhs); } }; +// This specialization enforces the use of a coefficient-based evaluation strategy +template<typename Lhs, typename Rhs> +struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,LazyCoeffBasedProductMode> + : generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,CoeffBasedProductMode> {}; + // Case 2: Evaluate coeff by coeff // // This is mostly taken from CoeffBasedProduct.h @@ -117,65 +366,116 @@ struct etor_product_coeff_impl; template<int StorageOrder, int UnrollingIndex, typename Lhs, typename Rhs, typename Packet, int LoadMode> struct etor_product_packet_impl; -template<typename Lhs, typename Rhs, typename LhsNested, typename RhsNested, int Flags> -struct product_evaluator_traits_dispatcher<Product<Lhs, Rhs>, CoeffBasedProduct<LhsNested, RhsNested, Flags> > +template<typename Lhs, typename Rhs, int ProductTag> +struct product_evaluator<Product<Lhs, Rhs, LazyProduct>, ProductTag, DenseShape, DenseShape, typename Lhs::Scalar, typename Rhs::Scalar > + : evaluator_base<Product<Lhs, Rhs, LazyProduct> > { - static const int HasEvalTo = 0; -}; - -template<typename Lhs, typename Rhs, typename LhsNested, typename RhsNested, int Flags> -struct product_evaluator_dispatcher<Product<Lhs, Rhs>, CoeffBasedProduct<LhsNested, RhsNested, Flags> > - : evaluator_impl_base<Product<Lhs, Rhs> > -{ - typedef Product<Lhs, Rhs> XprType; - typedef CoeffBasedProduct<LhsNested, RhsNested, Flags> CoeffBasedProductType; - - product_evaluator_dispatcher(const XprType& xpr) - : m_lhsImpl(xpr.lhs()), - m_rhsImpl(xpr.rhs()), - m_innerDim(xpr.lhs().cols()) - { } - + typedef Product<Lhs, Rhs, LazyProduct> XprType; typedef typename XprType::Index Index; typedef typename XprType::Scalar Scalar; typedef typename XprType::CoeffReturnType CoeffReturnType; typedef typename XprType::PacketScalar PacketScalar; typedef typename XprType::PacketReturnType PacketReturnType; + product_evaluator(const XprType& xpr) + : m_lhs(xpr.lhs()), + m_rhs(xpr.rhs()), + m_lhsImpl(m_lhs), // FIXME the creation of the evaluator objects should result in a no-op, but check that! + m_rhsImpl(m_rhs), // Moreover, they are only useful for the packet path, so we could completely disable them when not needed, + // or perhaps declare them on the fly on the packet method... We have experiment to check what's best. + m_innerDim(xpr.lhs().cols()) + { } + // Everything below here is taken from CoeffBasedProduct.h + typedef typename internal::nested_eval<Lhs,Rhs::ColsAtCompileTime>::type LhsNested; + typedef typename internal::nested_eval<Rhs,Lhs::RowsAtCompileTime>::type RhsNested; + + typedef typename internal::remove_all<LhsNested>::type LhsNestedCleaned; + typedef typename internal::remove_all<RhsNested>::type RhsNestedCleaned; + + typedef typename evaluator<LhsNestedCleaned>::type LhsEtorType; + typedef typename evaluator<RhsNestedCleaned>::type RhsEtorType; + enum { - RowsAtCompileTime = traits<CoeffBasedProductType>::RowsAtCompileTime, + RowsAtCompileTime = LhsNestedCleaned::RowsAtCompileTime, + ColsAtCompileTime = RhsNestedCleaned::ColsAtCompileTime, + InnerSize = EIGEN_SIZE_MIN_PREFER_FIXED(LhsNestedCleaned::ColsAtCompileTime, RhsNestedCleaned::RowsAtCompileTime), + MaxRowsAtCompileTime = LhsNestedCleaned::MaxRowsAtCompileTime, + MaxColsAtCompileTime = RhsNestedCleaned::MaxColsAtCompileTime, + PacketSize = packet_traits<Scalar>::size, - InnerSize = traits<CoeffBasedProductType>::InnerSize, - CoeffReadCost = traits<CoeffBasedProductType>::CoeffReadCost, + + LhsCoeffReadCost = LhsEtorType::CoeffReadCost, + RhsCoeffReadCost = RhsEtorType::CoeffReadCost, + CoeffReadCost = (InnerSize == Dynamic || LhsCoeffReadCost==Dynamic || RhsCoeffReadCost==Dynamic || NumTraits<Scalar>::AddCost==Dynamic || NumTraits<Scalar>::MulCost==Dynamic) ? Dynamic + : InnerSize * (NumTraits<Scalar>::MulCost + LhsCoeffReadCost + RhsCoeffReadCost) + + (InnerSize - 1) * NumTraits<Scalar>::AddCost, + Unroll = CoeffReadCost != Dynamic && CoeffReadCost <= EIGEN_UNROLLING_LIMIT, - CanVectorizeInner = traits<CoeffBasedProductType>::CanVectorizeInner + + LhsFlags = LhsEtorType::Flags, + RhsFlags = RhsEtorType::Flags, + + LhsRowMajor = LhsFlags & RowMajorBit, + RhsRowMajor = RhsFlags & RowMajorBit, + + SameType = is_same<typename LhsNestedCleaned::Scalar,typename RhsNestedCleaned::Scalar>::value, + + CanVectorizeRhs = RhsRowMajor && (RhsFlags & PacketAccessBit) + && (ColsAtCompileTime == Dynamic + || ( (ColsAtCompileTime % packet_traits<Scalar>::size) == 0 + && (RhsFlags&AlignedBit) + ) + ), + + CanVectorizeLhs = (!LhsRowMajor) && (LhsFlags & PacketAccessBit) + && (RowsAtCompileTime == Dynamic + || ( (RowsAtCompileTime % packet_traits<Scalar>::size) == 0 + && (LhsFlags&AlignedBit) + ) + ), + + EvalToRowMajor = (MaxRowsAtCompileTime==1&&MaxColsAtCompileTime!=1) ? 1 + : (MaxColsAtCompileTime==1&&MaxRowsAtCompileTime!=1) ? 0 + : (RhsRowMajor && !CanVectorizeLhs), + + Flags = ((unsigned int)(LhsFlags | RhsFlags) & HereditaryBits & ~RowMajorBit) + | (EvalToRowMajor ? RowMajorBit : 0) + | (CanVectorizeLhs ? (LhsFlags & AlignedBit) : 0) + | (CanVectorizeRhs ? (RhsFlags & AlignedBit) : 0) + // TODO enable vectorization for mixed types + | (SameType && (CanVectorizeLhs || CanVectorizeRhs) ? PacketAccessBit : 0), + + /* CanVectorizeInner deserves special explanation. It does not affect the product flags. It is not used outside + * of Product. If the Product itself is not a packet-access expression, there is still a chance that the inner + * loop of the product might be vectorized. This is the meaning of CanVectorizeInner. Since it doesn't affect + * the Flags, it is safe to make this value depend on ActualPacketAccessBit, that doesn't affect the ABI. + */ + CanVectorizeInner = SameType + && LhsRowMajor + && (!RhsRowMajor) + && (LhsFlags & RhsFlags & ActualPacketAccessBit) + && (LhsFlags & RhsFlags & AlignedBit) + && (InnerSize % packet_traits<Scalar>::size == 0) }; - - typedef typename evaluator<Lhs>::type LhsEtorType; - typedef typename evaluator<Rhs>::type RhsEtorType; - typedef etor_product_coeff_impl<CanVectorizeInner ? InnerVectorizedTraversal : DefaultTraversal, - Unroll ? InnerSize-1 : Dynamic, - LhsEtorType, RhsEtorType, Scalar> CoeffImpl; - + const CoeffReturnType coeff(Index row, Index col) const { - Scalar res; - CoeffImpl::run(row, col, m_lhsImpl, m_rhsImpl, m_innerDim, res); - return res; + // TODO check performance regression wrt to Eigen 3.2 which has special handling of this function + return (m_lhs.row(row).transpose().cwiseProduct( m_rhs.col(col) )).sum(); } /* Allow index-based non-packet access. It is impossible though to allow index-based packed access, * which is why we don't set the LinearAccessBit. + * TODO: this seems possible when the result is a vector */ const CoeffReturnType coeff(Index index) const { - Scalar res; const Index row = RowsAtCompileTime == 1 ? 0 : index; const Index col = RowsAtCompileTime == 1 ? index : 0; - CoeffImpl::run(row, col, m_lhsImpl, m_rhsImpl, m_innerDim, res); - return res; + // TODO check performance regression wrt to Eigen 3.2 which has special handling of this function + return (m_lhs.row(row).transpose().cwiseProduct( m_rhs.col(col) )).sum(); } template<int LoadMode> @@ -183,224 +483,376 @@ struct product_evaluator_dispatcher<Product<Lhs, Rhs>, CoeffBasedProduct<LhsNest { PacketScalar res; typedef etor_product_packet_impl<Flags&RowMajorBit ? RowMajor : ColMajor, - Unroll ? InnerSize-1 : Dynamic, - LhsEtorType, RhsEtorType, PacketScalar, LoadMode> PacketImpl; + Unroll ? InnerSize-1 : Dynamic, + LhsEtorType, RhsEtorType, PacketScalar, LoadMode> PacketImpl; + PacketImpl::run(row, col, m_lhsImpl, m_rhsImpl, m_innerDim, res); return res; } protected: - typename evaluator<Lhs>::type m_lhsImpl; - typename evaluator<Rhs>::type m_rhsImpl; + const LhsNested m_lhs; + const RhsNested m_rhs; + + LhsEtorType m_lhsImpl; + RhsEtorType m_rhsImpl; // TODO: Get rid of m_innerDim if known at compile time Index m_innerDim; }; -/*************************************************************************** -* Normal product .coeff() implementation (with meta-unrolling) -***************************************************************************/ +template<typename Lhs, typename Rhs> +struct product_evaluator<Product<Lhs, Rhs, DefaultProduct>, LazyCoeffBasedProductMode, DenseShape, DenseShape, typename traits<Lhs>::Scalar, typename traits<Rhs>::Scalar > + : product_evaluator<Product<Lhs, Rhs, LazyProduct>, CoeffBasedProductMode, DenseShape, DenseShape, typename traits<Lhs>::Scalar, typename traits<Rhs>::Scalar > +{ + typedef Product<Lhs, Rhs, DefaultProduct> XprType; + typedef Product<Lhs, Rhs, LazyProduct> BaseProduct; + typedef product_evaluator<BaseProduct, CoeffBasedProductMode, DenseShape, DenseShape, typename Lhs::Scalar, typename Rhs::Scalar > Base; + product_evaluator(const XprType& xpr) + : Base(BaseProduct(xpr.lhs(),xpr.rhs())) + {} +}; -/************************************** -*** Scalar path - no vectorization *** -**************************************/ +/**************************************** +*** Coeff based product, Packet path *** +****************************************/ -template<int UnrollingIndex, typename Lhs, typename Rhs, typename RetScalar> -struct etor_product_coeff_impl<DefaultTraversal, UnrollingIndex, Lhs, Rhs, RetScalar> +template<int UnrollingIndex, typename Lhs, typename Rhs, typename Packet, int LoadMode> +struct etor_product_packet_impl<RowMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode> { typedef typename Lhs::Index Index; - static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, RetScalar &res) + static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet &res) { - etor_product_coeff_impl<DefaultTraversal, UnrollingIndex-1, Lhs, Rhs, RetScalar>::run(row, col, lhs, rhs, innerDim, res); - res += lhs.coeff(row, UnrollingIndex) * rhs.coeff(UnrollingIndex, col); + etor_product_packet_impl<RowMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, innerDim, res); + res = pmadd(pset1<Packet>(lhs.coeff(row, UnrollingIndex)), rhs.template packet<LoadMode>(UnrollingIndex, col), res); } }; -template<typename Lhs, typename Rhs, typename RetScalar> -struct etor_product_coeff_impl<DefaultTraversal, 0, Lhs, Rhs, RetScalar> +template<int UnrollingIndex, typename Lhs, typename Rhs, typename Packet, int LoadMode> +struct etor_product_packet_impl<ColMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode> { typedef typename Lhs::Index Index; - static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index /*innerDim*/, RetScalar &res) + static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet &res) { - res = lhs.coeff(row, 0) * rhs.coeff(0, col); + etor_product_packet_impl<ColMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, innerDim, res); + res = pmadd(lhs.template packet<LoadMode>(row, UnrollingIndex), pset1<Packet>(rhs.coeff(UnrollingIndex, col)), res); } }; -template<typename Lhs, typename Rhs, typename RetScalar> -struct etor_product_coeff_impl<DefaultTraversal, Dynamic, Lhs, Rhs, RetScalar> +template<typename Lhs, typename Rhs, typename Packet, int LoadMode> +struct etor_product_packet_impl<RowMajor, 0, Lhs, Rhs, Packet, LoadMode> { typedef typename Lhs::Index Index; - static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, RetScalar& res) + static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index /*innerDim*/, Packet &res) { - eigen_assert(innerDim>0 && "you are using a non initialized matrix"); - res = lhs.coeff(row, 0) * rhs.coeff(0, col); - for(Index i = 1; i < innerDim; ++i) - res += lhs.coeff(row, i) * rhs.coeff(i, col); + res = pmul(pset1<Packet>(lhs.coeff(row, 0)),rhs.template packet<LoadMode>(0, col)); } }; -/******************************************* -*** Scalar path with inner vectorization *** -*******************************************/ - -template<int UnrollingIndex, typename Lhs, typename Rhs, typename Packet> -struct etor_product_coeff_vectorized_unroller +template<typename Lhs, typename Rhs, typename Packet, int LoadMode> +struct etor_product_packet_impl<ColMajor, 0, Lhs, Rhs, Packet, LoadMode> { typedef typename Lhs::Index Index; - enum { PacketSize = packet_traits<typename Lhs::Scalar>::size }; - static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, typename Lhs::PacketScalar &pres) + static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index /*innerDim*/, Packet &res) { - etor_product_coeff_vectorized_unroller<UnrollingIndex-PacketSize, Lhs, Rhs, Packet>::run(row, col, lhs, rhs, innerDim, pres); - pres = padd(pres, pmul( lhs.template packet<Aligned>(row, UnrollingIndex) , rhs.template packet<Aligned>(UnrollingIndex, col) )); + res = pmul(lhs.template packet<LoadMode>(row, 0), pset1<Packet>(rhs.coeff(0, col))); } }; -template<typename Lhs, typename Rhs, typename Packet> -struct etor_product_coeff_vectorized_unroller<0, Lhs, Rhs, Packet> +template<typename Lhs, typename Rhs, typename Packet, int LoadMode> +struct etor_product_packet_impl<RowMajor, Dynamic, Lhs, Rhs, Packet, LoadMode> { typedef typename Lhs::Index Index; - static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index /*innerDim*/, typename Lhs::PacketScalar &pres) + static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet& res) { - pres = pmul(lhs.template packet<Aligned>(row, 0) , rhs.template packet<Aligned>(0, col)); + eigen_assert(innerDim>0 && "you are using a non initialized matrix"); + res = pmul(pset1<Packet>(lhs.coeff(row, 0)),rhs.template packet<LoadMode>(0, col)); + for(Index i = 1; i < innerDim; ++i) + res = pmadd(pset1<Packet>(lhs.coeff(row, i)), rhs.template packet<LoadMode>(i, col), res); } }; -template<int UnrollingIndex, typename Lhs, typename Rhs, typename RetScalar> -struct etor_product_coeff_impl<InnerVectorizedTraversal, UnrollingIndex, Lhs, Rhs, RetScalar> +template<typename Lhs, typename Rhs, typename Packet, int LoadMode> +struct etor_product_packet_impl<ColMajor, Dynamic, Lhs, Rhs, Packet, LoadMode> { - typedef typename Lhs::PacketScalar Packet; typedef typename Lhs::Index Index; - enum { PacketSize = packet_traits<typename Lhs::Scalar>::size }; - static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, RetScalar &res) + static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet& res) { - Packet pres; - etor_product_coeff_vectorized_unroller<UnrollingIndex+1-PacketSize, Lhs, Rhs, Packet>::run(row, col, lhs, rhs, innerDim, pres); - etor_product_coeff_impl<DefaultTraversal,UnrollingIndex,Lhs,Rhs,RetScalar>::run(row, col, lhs, rhs, innerDim, res); - res = predux(pres); + eigen_assert(innerDim>0 && "you are using a non initialized matrix"); + res = pmul(lhs.template packet<LoadMode>(row, 0), pset1<Packet>(rhs.coeff(0, col))); + for(Index i = 1; i < innerDim; ++i) + res = pmadd(lhs.template packet<LoadMode>(row, i), pset1<Packet>(rhs.coeff(i, col)), res); } }; -template<typename Lhs, typename Rhs, int LhsRows = Lhs::RowsAtCompileTime, int RhsCols = Rhs::ColsAtCompileTime> -struct etor_product_coeff_vectorized_dyn_selector + +/*************************************************************************** +* Triangular products +***************************************************************************/ +template<int Mode, bool LhsIsTriangular, + typename Lhs, bool LhsIsVector, + typename Rhs, bool RhsIsVector> +struct triangular_product_impl; + +template<typename Lhs, typename Rhs, int ProductTag> +struct generic_product_impl<Lhs,Rhs,TriangularShape,DenseShape,ProductTag> + : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,TriangularShape,DenseShape,ProductTag> > { - typedef typename Lhs::Index Index; - static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index /*innerDim*/, typename Lhs::Scalar &res) + typedef typename Product<Lhs,Rhs>::Scalar Scalar; + + template<typename Dest> + static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) { - res = lhs.row(row).transpose().cwiseProduct(rhs.col(col)).sum(); + triangular_product_impl<Lhs::Mode,true,typename Lhs::MatrixType,false,Rhs, Rhs::ColsAtCompileTime==1> + ::run(dst, lhs.nestedExpression(), rhs, alpha); } }; -// NOTE the 3 following specializations are because taking .col(0) on a vector is a bit slower -// NOTE maybe they are now useless since we have a specialization for Block<Matrix> -template<typename Lhs, typename Rhs, int RhsCols> -struct etor_product_coeff_vectorized_dyn_selector<Lhs,Rhs,1,RhsCols> +template<typename Lhs, typename Rhs, int ProductTag> +struct generic_product_impl<Lhs,Rhs,DenseShape,TriangularShape,ProductTag> +: generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,DenseShape,TriangularShape,ProductTag> > { - typedef typename Lhs::Index Index; - static EIGEN_STRONG_INLINE void run(Index /*row*/, Index col, const Lhs& lhs, const Rhs& rhs, Index /*innerDim*/, typename Lhs::Scalar &res) + typedef typename Product<Lhs,Rhs>::Scalar Scalar; + + template<typename Dest> + static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) { - res = lhs.transpose().cwiseProduct(rhs.col(col)).sum(); + triangular_product_impl<Rhs::Mode,false,Lhs,Lhs::RowsAtCompileTime==1, typename Rhs::MatrixType, false>::run(dst, lhs, rhs.nestedExpression(), alpha); } }; -template<typename Lhs, typename Rhs, int LhsRows> -struct etor_product_coeff_vectorized_dyn_selector<Lhs,Rhs,LhsRows,1> + +/*************************************************************************** +* SelfAdjoint products +***************************************************************************/ +template <typename Lhs, int LhsMode, bool LhsIsVector, + typename Rhs, int RhsMode, bool RhsIsVector> +struct selfadjoint_product_impl; + +template<typename Lhs, typename Rhs, int ProductTag> +struct generic_product_impl<Lhs,Rhs,SelfAdjointShape,DenseShape,ProductTag> + : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,SelfAdjointShape,DenseShape,ProductTag> > { - typedef typename Lhs::Index Index; - static EIGEN_STRONG_INLINE void run(Index row, Index /*col*/, const Lhs& lhs, const Rhs& rhs, Index /*innerDim*/, typename Lhs::Scalar &res) + typedef typename Product<Lhs,Rhs>::Scalar Scalar; + + template<typename Dest> + static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) { - res = lhs.row(row).transpose().cwiseProduct(rhs).sum(); + selfadjoint_product_impl<typename Lhs::MatrixType,Lhs::Mode,false,Rhs,0,Rhs::IsVectorAtCompileTime>::run(dst, lhs.nestedExpression(), rhs, alpha); } }; -template<typename Lhs, typename Rhs> -struct etor_product_coeff_vectorized_dyn_selector<Lhs,Rhs,1,1> +template<typename Lhs, typename Rhs, int ProductTag> +struct generic_product_impl<Lhs,Rhs,DenseShape,SelfAdjointShape,ProductTag> +: generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,DenseShape,SelfAdjointShape,ProductTag> > { - typedef typename Lhs::Index Index; - EIGEN_STRONG_INLINE void run(Index /*row*/, Index /*col*/, const Lhs& lhs, const Rhs& rhs, Index /*innerDim*/, typename Lhs::Scalar &res) + typedef typename Product<Lhs,Rhs>::Scalar Scalar; + + template<typename Dest> + static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) { - res = lhs.transpose().cwiseProduct(rhs).sum(); + selfadjoint_product_impl<Lhs,0,Lhs::IsVectorAtCompileTime,typename Rhs::MatrixType,Rhs::Mode,false>::run(dst, lhs, rhs.nestedExpression(), alpha); } }; -template<typename Lhs, typename Rhs, typename RetScalar> -struct etor_product_coeff_impl<InnerVectorizedTraversal, Dynamic, Lhs, Rhs, RetScalar> + +/*************************************************************************** +* Diagonal products +***************************************************************************/ + +template<typename MatrixType, typename DiagonalType, typename Derived, int ProductOrder> +struct diagonal_product_evaluator_base + : evaluator_base<Derived> { - typedef typename Lhs::Index Index; - static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, typename Lhs::Scalar &res) + typedef typename MatrixType::Index Index; + typedef typename scalar_product_traits<typename MatrixType::Scalar, typename DiagonalType::Scalar>::ReturnType Scalar; + typedef typename internal::packet_traits<Scalar>::type PacketScalar; +public: + enum { + CoeffReadCost = NumTraits<Scalar>::MulCost + evaluator<MatrixType>::CoeffReadCost + evaluator<DiagonalType>::CoeffReadCost, + + MatrixFlags = evaluator<MatrixType>::Flags, + DiagFlags = evaluator<DiagonalType>::Flags, + _StorageOrder = MatrixFlags & RowMajorBit ? RowMajor : ColMajor, + _ScalarAccessOnDiag = !((int(_StorageOrder) == ColMajor && int(ProductOrder) == OnTheLeft) + ||(int(_StorageOrder) == RowMajor && int(ProductOrder) == OnTheRight)), + _SameTypes = is_same<typename MatrixType::Scalar, typename DiagonalType::Scalar>::value, + // FIXME currently we need same types, but in the future the next rule should be the one + //_Vectorizable = bool(int(MatrixFlags)&PacketAccessBit) && ((!_PacketOnDiag) || (_SameTypes && bool(int(DiagFlags)&PacketAccessBit))), + _Vectorizable = bool(int(MatrixFlags)&PacketAccessBit) && _SameTypes && (_ScalarAccessOnDiag || (bool(int(DiagFlags)&PacketAccessBit))), + _LinearAccessMask = (MatrixType::RowsAtCompileTime==1 || MatrixType::ColsAtCompileTime==1) ? LinearAccessBit : 0, + Flags = ((HereditaryBits|_LinearAccessMask) & (unsigned int)(MatrixFlags)) | (_Vectorizable ? PacketAccessBit : 0) | AlignedBit + //(int(MatrixFlags)&int(DiagFlags)&AlignedBit), + }; + + diagonal_product_evaluator_base(const MatrixType &mat, const DiagonalType &diag) + : m_diagImpl(diag), m_matImpl(mat) + { + } + + EIGEN_STRONG_INLINE const Scalar coeff(Index idx) const { - etor_product_coeff_vectorized_dyn_selector<Lhs,Rhs>::run(row, col, lhs, rhs, innerDim, res); + return m_diagImpl.coeff(idx) * m_matImpl.coeff(idx); } + +protected: + template<int LoadMode> + EIGEN_STRONG_INLINE PacketScalar packet_impl(Index row, Index col, Index id, internal::true_type) const + { + return internal::pmul(m_matImpl.template packet<LoadMode>(row, col), + internal::pset1<PacketScalar>(m_diagImpl.coeff(id))); + } + + template<int LoadMode> + EIGEN_STRONG_INLINE PacketScalar packet_impl(Index row, Index col, Index id, internal::false_type) const + { + enum { + InnerSize = (MatrixType::Flags & RowMajorBit) ? MatrixType::ColsAtCompileTime : MatrixType::RowsAtCompileTime, + DiagonalPacketLoadMode = (LoadMode == Aligned && (((InnerSize%16) == 0) || (int(DiagFlags)&AlignedBit)==AlignedBit) ? Aligned : Unaligned) + }; + return internal::pmul(m_matImpl.template packet<LoadMode>(row, col), + m_diagImpl.template packet<DiagonalPacketLoadMode>(id)); + } + + typename evaluator<DiagonalType>::nestedType m_diagImpl; + typename evaluator<MatrixType>::nestedType m_matImpl; }; -/******************* -*** Packet path *** -*******************/ - -template<int UnrollingIndex, typename Lhs, typename Rhs, typename Packet, int LoadMode> -struct etor_product_packet_impl<RowMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode> +// diagonal * dense +template<typename Lhs, typename Rhs, int ProductKind, int ProductTag> +struct product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, DiagonalShape, DenseShape, typename Lhs::Scalar, typename Rhs::Scalar> + : diagonal_product_evaluator_base<Rhs, typename Lhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>, OnTheLeft> { - typedef typename Lhs::Index Index; - static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet &res) + typedef diagonal_product_evaluator_base<Rhs, typename Lhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>, OnTheLeft> Base; + using Base::m_diagImpl; + using Base::m_matImpl; + using Base::coeff; + using Base::packet_impl; + typedef typename Base::Scalar Scalar; + typedef typename Base::Index Index; + typedef typename Base::PacketScalar PacketScalar; + + typedef Product<Lhs, Rhs, ProductKind> XprType; + typedef typename XprType::PlainObject PlainObject; + + enum { + StorageOrder = int(Rhs::Flags) & RowMajorBit ? RowMajor : ColMajor + }; + + product_evaluator(const XprType& xpr) + : Base(xpr.rhs(), xpr.lhs().diagonal()) { - etor_product_packet_impl<RowMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, innerDim, res); - res = pmadd(pset1<Packet>(lhs.coeff(row, UnrollingIndex)), rhs.template packet<LoadMode>(UnrollingIndex, col), res); } + + EIGEN_STRONG_INLINE const Scalar coeff(Index row, Index col) const + { + return m_diagImpl.coeff(row) * m_matImpl.coeff(row, col); + } + + template<int LoadMode> + EIGEN_STRONG_INLINE PacketScalar packet(Index row, Index col) const + { + return this->template packet_impl<LoadMode>(row,col, row, + typename internal::conditional<int(StorageOrder)==RowMajor, internal::true_type, internal::false_type>::type()); + } + + template<int LoadMode> + EIGEN_STRONG_INLINE PacketScalar packet(Index idx) const + { + return packet<LoadMode>(int(StorageOrder)==ColMajor?idx:0,int(StorageOrder)==ColMajor?0:idx); + } + }; -template<int UnrollingIndex, typename Lhs, typename Rhs, typename Packet, int LoadMode> -struct etor_product_packet_impl<ColMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode> +// dense * diagonal +template<typename Lhs, typename Rhs, int ProductKind, int ProductTag> +struct product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, DenseShape, DiagonalShape, typename Lhs::Scalar, typename Rhs::Scalar> + : diagonal_product_evaluator_base<Lhs, typename Rhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>, OnTheRight> { - typedef typename Lhs::Index Index; - static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet &res) + typedef diagonal_product_evaluator_base<Lhs, typename Rhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>, OnTheRight> Base; + using Base::m_diagImpl; + using Base::m_matImpl; + using Base::coeff; + using Base::packet_impl; + typedef typename Base::Scalar Scalar; + typedef typename Base::Index Index; + typedef typename Base::PacketScalar PacketScalar; + + typedef Product<Lhs, Rhs, ProductKind> XprType; + typedef typename XprType::PlainObject PlainObject; + + enum { StorageOrder = int(Lhs::Flags) & RowMajorBit ? RowMajor : ColMajor }; + + product_evaluator(const XprType& xpr) + : Base(xpr.lhs(), xpr.rhs().diagonal()) { - etor_product_packet_impl<ColMajor, UnrollingIndex-1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs, innerDim, res); - res = pmadd(lhs.template packet<LoadMode>(row, UnrollingIndex), pset1<Packet>(rhs.coeff(UnrollingIndex, col)), res); } + + EIGEN_STRONG_INLINE const Scalar coeff(Index row, Index col) const + { + return m_matImpl.coeff(row, col) * m_diagImpl.coeff(col); + } + + template<int LoadMode> + EIGEN_STRONG_INLINE PacketScalar packet(Index row, Index col) const + { + return this->template packet_impl<LoadMode>(row,col, col, + typename internal::conditional<int(StorageOrder)==ColMajor, internal::true_type, internal::false_type>::type()); + } + + template<int LoadMode> + EIGEN_STRONG_INLINE PacketScalar packet(Index idx) const + { + return packet<LoadMode>(int(StorageOrder)==ColMajor?idx:0,int(StorageOrder)==ColMajor?0:idx); + } + }; -template<typename Lhs, typename Rhs, typename Packet, int LoadMode> -struct etor_product_packet_impl<RowMajor, 0, Lhs, Rhs, Packet, LoadMode> +/*************************************************************************** +* Products with permutation matrices +***************************************************************************/ + +template<typename Lhs, typename Rhs, int ProductTag> +struct generic_product_impl<Lhs, Rhs, PermutationShape, DenseShape, ProductTag> { - typedef typename Lhs::Index Index; - static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index /*innerDim*/, Packet &res) + template<typename Dest> + static void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) { - res = pmul(pset1<Packet>(lhs.coeff(row, 0)),rhs.template packet<LoadMode>(0, col)); + permut_matrix_product_retval<Lhs, Rhs, OnTheLeft, false> pmpr(lhs, rhs); + pmpr.evalTo(dst); } }; -template<typename Lhs, typename Rhs, typename Packet, int LoadMode> -struct etor_product_packet_impl<ColMajor, 0, Lhs, Rhs, Packet, LoadMode> +template<typename Lhs, typename Rhs, int ProductTag> +struct generic_product_impl<Lhs, Rhs, DenseShape, PermutationShape, ProductTag> { - typedef typename Lhs::Index Index; - static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index /*innerDim*/, Packet &res) + template<typename Dest> + static void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) { - res = pmul(lhs.template packet<LoadMode>(row, 0), pset1<Packet>(rhs.coeff(0, col))); + permut_matrix_product_retval<Rhs, Lhs, OnTheRight, false> pmpr(rhs, lhs); + pmpr.evalTo(dst); } }; -template<typename Lhs, typename Rhs, typename Packet, int LoadMode> -struct etor_product_packet_impl<RowMajor, Dynamic, Lhs, Rhs, Packet, LoadMode> +template<typename Lhs, typename Rhs, int ProductTag> +struct generic_product_impl<Transpose<Lhs>, Rhs, PermutationShape, DenseShape, ProductTag> { - typedef typename Lhs::Index Index; - static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet& res) + template<typename Dest> + static void evalTo(Dest& dst, const Transpose<Lhs>& lhs, const Rhs& rhs) { - eigen_assert(innerDim>0 && "you are using a non initialized matrix"); - res = pmul(pset1<Packet>(lhs.coeff(row, 0)),rhs.template packet<LoadMode>(0, col)); - for(Index i = 1; i < innerDim; ++i) - res = pmadd(pset1<Packet>(lhs.coeff(row, i)), rhs.template packet<LoadMode>(i, col), res); + permut_matrix_product_retval<Lhs, Rhs, OnTheLeft, true> pmpr(lhs.nestedPermutation(), rhs); + pmpr.evalTo(dst); } }; -template<typename Lhs, typename Rhs, typename Packet, int LoadMode> -struct etor_product_packet_impl<ColMajor, Dynamic, Lhs, Rhs, Packet, LoadMode> +template<typename Lhs, typename Rhs, int ProductTag> +struct generic_product_impl<Lhs, Transpose<Rhs>, DenseShape, PermutationShape, ProductTag> { - typedef typename Lhs::Index Index; - static EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs, Index innerDim, Packet& res) + template<typename Dest> + static void evalTo(Dest& dst, const Lhs& lhs, const Transpose<Rhs>& rhs) { - eigen_assert(innerDim>0 && "you are using a non initialized matrix"); - res = pmul(lhs.template packet<LoadMode>(row, 0), pset1<Packet>(rhs.coeff(0, col))); - for(Index i = 1; i < innerDim; ++i) - res = pmadd(lhs.template packet<LoadMode>(row, i), pset1<Packet>(rhs.coeff(i, col)), res); + permut_matrix_product_retval<Rhs, Lhs, OnTheRight, true> pmpr(rhs.nestedPermutation(), lhs); + pmpr.evalTo(dst); } }; diff --git a/Eigen/src/Core/Redux.h b/Eigen/src/Core/Redux.h index 5b82c9a65..136cd165e 100644 --- a/Eigen/src/Core/Redux.h +++ b/Eigen/src/Core/Redux.h @@ -65,6 +65,29 @@ public: ? CompleteUnrolling : NoUnrolling }; + +#ifdef EIGEN_DEBUG_ASSIGN + static void debug() + { +#ifdef EIGEN_TEST_EVALUATORS + std::cerr << "Xpr: " << typeid(typename Derived::XprType).name() << std::endl; +#else + std::cerr << "Xpr: " << typeid(Derived).name() << std::endl; +#endif + std::cerr.setf(std::ios::hex, std::ios::basefield); + EIGEN_DEBUG_VAR(Derived::Flags) + std::cerr.unsetf(std::ios::hex); + EIGEN_DEBUG_VAR(InnerMaxSize) + EIGEN_DEBUG_VAR(PacketSize) + EIGEN_DEBUG_VAR(MightVectorize) + EIGEN_DEBUG_VAR(MayLinearVectorize) + EIGEN_DEBUG_VAR(MaySliceVectorize) + EIGEN_DEBUG_VAR(Traversal) + EIGEN_DEBUG_VAR(UnrollingLimit) + EIGEN_DEBUG_VAR(Unrolling) + std::cerr << std::endl; + } +#endif }; /*************************************************************************** @@ -174,7 +197,7 @@ struct redux_impl<Func, Derived, DefaultTraversal, NoUnrolling> typedef typename Derived::Scalar Scalar; typedef typename Derived::Index Index; EIGEN_DEVICE_FUNC - static EIGEN_STRONG_INLINE Scalar run(const Derived& mat, const Func& func) + static EIGEN_STRONG_INLINE Scalar run(const Derived &mat, const Func& func) { eigen_assert(mat.rows()>0 && mat.cols()>0 && "you are using an empty matrix"); Scalar res; @@ -200,10 +223,10 @@ struct redux_impl<Func, Derived, LinearVectorizedTraversal, NoUnrolling> typedef typename packet_traits<Scalar>::type PacketScalar; typedef typename Derived::Index Index; - static Scalar run(const Derived& mat, const Func& func) + static Scalar run(const Derived &mat, const Func& func) { const Index size = mat.size(); - eigen_assert(size && "you are using an empty matrix"); + const Index packetSize = packet_traits<Scalar>::size; const Index alignedStart = internal::first_aligned(mat); enum { @@ -258,7 +281,7 @@ struct redux_impl<Func, Derived, SliceVectorizedTraversal, NoUnrolling> typedef typename packet_traits<Scalar>::type PacketScalar; typedef typename Derived::Index Index; - static Scalar run(const Derived& mat, const Func& func) + static Scalar run(const Derived &mat, const Func& func) { eigen_assert(mat.rows()>0 && mat.cols()>0 && "you are using an empty matrix"); const Index innerSize = mat.innerSize(); @@ -300,7 +323,7 @@ struct redux_impl<Func, Derived, LinearVectorizedTraversal, CompleteUnrolling> Size = Derived::SizeAtCompileTime, VectorizedSize = (Size / PacketSize) * PacketSize }; - static EIGEN_STRONG_INLINE Scalar run(const Derived& mat, const Func& func) + static EIGEN_STRONG_INLINE Scalar run(const Derived &mat, const Func& func) { eigen_assert(mat.rows()>0 && mat.cols()>0 && "you are using an empty matrix"); if (VectorizedSize > 0) { @@ -315,6 +338,65 @@ struct redux_impl<Func, Derived, LinearVectorizedTraversal, CompleteUnrolling> } }; +#ifdef EIGEN_ENABLE_EVALUATORS +// evaluator adaptor +template<typename _XprType> +class redux_evaluator +{ +public: + typedef _XprType XprType; + redux_evaluator(const XprType &xpr) : m_evaluator(xpr), m_xpr(xpr) {} + + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + typedef typename XprType::PacketScalar PacketScalar; + typedef typename XprType::PacketReturnType PacketReturnType; + + enum { + MaxRowsAtCompileTime = XprType::MaxRowsAtCompileTime, + MaxColsAtCompileTime = XprType::MaxColsAtCompileTime, + // TODO we should not remove DirectAccessBit and rather find an elegant way to query the alignment offset at runtime from the evaluator + Flags = evaluator<XprType>::Flags & ~DirectAccessBit, + IsRowMajor = XprType::IsRowMajor, + SizeAtCompileTime = XprType::SizeAtCompileTime, + InnerSizeAtCompileTime = XprType::InnerSizeAtCompileTime, + CoeffReadCost = evaluator<XprType>::CoeffReadCost + }; + + Index rows() const { return m_xpr.rows(); } + Index cols() const { return m_xpr.cols(); } + Index size() const { return m_xpr.size(); } + Index innerSize() const { return m_xpr.innerSize(); } + Index outerSize() const { return m_xpr.outerSize(); } + + CoeffReturnType coeff(Index row, Index col) const + { return m_evaluator.coeff(row, col); } + + CoeffReturnType coeff(Index index) const + { return m_evaluator.coeff(index); } + + template<int LoadMode> + PacketReturnType packet(Index row, Index col) const + { return m_evaluator.template packet<LoadMode>(row, col); } + + template<int LoadMode> + PacketReturnType packet(Index index) const + { return m_evaluator.template packet<LoadMode>(index); } + + CoeffReturnType coeffByOuterInner(Index outer, Index inner) const + { return m_evaluator.coeff(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); } + + template<int LoadMode> + PacketReturnType packetByOuterInner(Index outer, Index inner) const + { return m_evaluator.template packet<LoadMode>(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); } + +protected: + typename internal::evaluator<XprType>::nestedType m_evaluator; + const XprType &m_xpr; +}; +#endif + } // end namespace internal /*************************************************************************** @@ -325,7 +407,7 @@ struct redux_impl<Func, Derived, LinearVectorizedTraversal, CompleteUnrolling> /** \returns the result of a full redux operation on the whole matrix or vector using \a func * * The template parameter \a BinaryOp is the type of the functor \a func which must be - * an associative operator. Both current STL and TR1 functor styles are handled. + * an associative operator. Both current C++98 and C++11 functor styles are handled. * * \sa DenseBase::sum(), DenseBase::minCoeff(), DenseBase::maxCoeff(), MatrixBase::colwise(), MatrixBase::rowwise() */ @@ -334,9 +416,30 @@ template<typename Func> EIGEN_STRONG_INLINE typename internal::result_of<Func(typename internal::traits<Derived>::Scalar)>::type DenseBase<Derived>::redux(const Func& func) const { + eigen_assert(this->rows()>0 && this->cols()>0 && "you are using an empty matrix"); +#ifdef EIGEN_TEST_EVALUATORS + + // FIXME, eval_nest should be handled by redux_evaluator, however: + // - it is currently difficult to provide the right Flags since they are still handled by the expressions + // - handling it here might reduce the number of template instantiations +// typedef typename internal::nested_eval<Derived,1>::type ThisNested; +// typedef typename internal::remove_all<ThisNested>::type ThisNestedCleaned; +// typedef typename internal::redux_evaluator<ThisNestedCleaned> ThisEvaluator; +// +// ThisNested thisNested(derived()); +// ThisEvaluator thisEval(thisNested); + + typedef typename internal::redux_evaluator<Derived> ThisEvaluator; + ThisEvaluator thisEval(derived()); + + return internal::redux_impl<Func, ThisEvaluator>::run(thisEval, func); + +#else typedef typename internal::remove_all<typename Derived::Nested>::type ThisNested; + return internal::redux_impl<Func, ThisNested> ::run(derived(), func); +#endif } /** \returns the minimum of all coefficients of \c *this. diff --git a/Eigen/src/Core/Ref.h b/Eigen/src/Core/Ref.h index 92614c6e2..6390a8b64 100644 --- a/Eigen/src/Core/Ref.h +++ b/Eigen/src/Core/Ref.h @@ -12,10 +12,6 @@ namespace Eigen { -template<typename Derived> class RefBase; -template<typename PlainObjectType, int Options = 0, - typename StrideType = typename internal::conditional<PlainObjectType::IsVectorAtCompileTime,InnerStride<1>,OuterStride<> >::type > class Ref; - /** \class Ref * \ingroup Core_Module * @@ -247,7 +243,11 @@ template<typename TPlainObjectType, int Options, typename StrideType> class Ref< template<typename Expression> void construct(const Expression& expr, internal::false_type) { +#ifdef EIGEN_TEST_EVALUATORS + internal::call_assignment_no_alias(m_object,expr,internal::assign_op<Scalar>()); +#else m_object.lazyAssign(expr); +#endif Base::construct(m_object); } diff --git a/Eigen/src/Core/Replicate.h b/Eigen/src/Core/Replicate.h index dde86a834..e63f0d421 100644 --- a/Eigen/src/Core/Replicate.h +++ b/Eigen/src/Core/Replicate.h @@ -53,8 +53,14 @@ struct traits<Replicate<MatrixType,RowFactor,ColFactor> > IsRowMajor = MaxRowsAtCompileTime==1 && MaxColsAtCompileTime!=1 ? 1 : MaxColsAtCompileTime==1 && MaxRowsAtCompileTime!=1 ? 0 : (MatrixType::Flags & RowMajorBit) ? 1 : 0, + +#ifndef EIGEN_TEST_EVALUATORS Flags = (_MatrixTypeNested::Flags & HereditaryBits & ~RowMajorBit) | (IsRowMajor ? RowMajorBit : 0), CoeffReadCost = _MatrixTypeNested::CoeffReadCost +#else + // FIXME enable DirectAccess with negative strides? + Flags = IsRowMajor ? RowMajorBit : 0 +#endif }; }; } @@ -68,6 +74,7 @@ template<typename MatrixType,int RowFactor,int ColFactor> class Replicate typedef typename internal::dense_xpr_base<Replicate>::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(Replicate) + typedef typename internal::remove_all<MatrixType>::type NestedExpression; template<typename OriginalMatrixType> inline explicit Replicate(const OriginalMatrixType& a_matrix) diff --git a/Eigen/src/Core/ReturnByValue.h b/Eigen/src/Core/ReturnByValue.h index 7834f6cbc..9d53fba27 100644 --- a/Eigen/src/Core/ReturnByValue.h +++ b/Eigen/src/Core/ReturnByValue.h @@ -33,6 +33,7 @@ struct traits<ReturnByValue<Derived> > }; }; +#ifndef EIGEN_TEST_EVALUATORS /* The ReturnByValue object doesn't even have a coeff() method. * So the only way that nesting it in an expression can work, is by evaluating it into a plain matrix. * So internal::nested always gives the plain return matrix type. @@ -44,6 +45,13 @@ struct nested<ReturnByValue<Derived>, n, PlainObject> { typedef typename traits<Derived>::ReturnType type; }; +#else +template<typename Derived,int n,typename PlainObject> +struct nested_eval<ReturnByValue<Derived>, n, PlainObject> +{ + typedef typename traits<Derived>::ReturnType type; +}; +#endif } // end namespace internal @@ -73,6 +81,7 @@ template<typename Derived> class ReturnByValue const Unusable& coeff(Index,Index) const { return *reinterpret_cast<const Unusable*>(this); } Unusable& coeffRef(Index) { return *reinterpret_cast<Unusable*>(this); } Unusable& coeffRef(Index,Index) { return *reinterpret_cast<Unusable*>(this); } +#undef Unusable #endif }; @@ -84,6 +93,38 @@ Derived& DenseBase<Derived>::operator=(const ReturnByValue<OtherDerived>& other) return derived(); } +#ifdef EIGEN_TEST_EVALUATORS +namespace internal { + +// Expression is evaluated in a temporary; default implementation of Assignment is bypassed so that +// when a ReturnByValue expression is assigned, the evaluator is not constructed. +// TODO: Finalize port to new regime; ReturnByValue should not exist in the expression world + +template<typename Derived> +struct evaluator<ReturnByValue<Derived> > + : public evaluator<typename internal::traits<Derived>::ReturnType>::type +{ + typedef ReturnByValue<Derived> XprType; + typedef typename internal::traits<Derived>::ReturnType PlainObject; + typedef typename evaluator<PlainObject>::type Base; + + typedef evaluator type; + typedef evaluator nestedType; + + evaluator(const XprType& xpr) + : m_result(xpr.rows(), xpr.cols()) + { + ::new (static_cast<Base*>(this)) Base(m_result); + xpr.evalTo(m_result); + } + +protected: + PlainObject m_result; +}; + +} // end namespace internal +#endif + } // end namespace Eigen #endif // EIGEN_RETURNBYVALUE_H diff --git a/Eigen/src/Core/Reverse.h b/Eigen/src/Core/Reverse.h index e30ae3d28..ceb6e4701 100644 --- a/Eigen/src/Core/Reverse.h +++ b/Eigen/src/Core/Reverse.h @@ -45,13 +45,16 @@ struct traits<Reverse<MatrixType, Direction> > MaxRowsAtCompileTime = MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxColsAtCompileTime, +#ifndef EIGEN_TEST_EVALUATORS // let's enable LinearAccess only with vectorization because of the product overhead LinearAccess = ( (Direction==BothDirections) && (int(_MatrixTypeNested::Flags)&PacketAccessBit) ) ? LinearAccessBit : 0, Flags = int(_MatrixTypeNested::Flags) & (HereditaryBits | LvalueBit | PacketAccessBit | LinearAccess), - CoeffReadCost = _MatrixTypeNested::CoeffReadCost +#else + Flags = _MatrixTypeNested::Flags & (RowMajorBit | LvalueBit) +#endif }; }; @@ -74,6 +77,7 @@ template<typename MatrixType, int Direction> class Reverse typedef typename internal::dense_xpr_base<Reverse>::type Base; EIGEN_DENSE_PUBLIC_INTERFACE(Reverse) + typedef typename internal::remove_all<MatrixType>::type NestedExpression; using Base::IsRowMajor; // next line is necessary because otherwise const version of operator() diff --git a/Eigen/src/Core/Select.h b/Eigen/src/Core/Select.h index 87993bbb5..d4fd88e62 100644 --- a/Eigen/src/Core/Select.h +++ b/Eigen/src/Core/Select.h @@ -43,10 +43,14 @@ struct traits<Select<ConditionMatrixType, ThenMatrixType, ElseMatrixType> > ColsAtCompileTime = ConditionMatrixType::ColsAtCompileTime, MaxRowsAtCompileTime = ConditionMatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = ConditionMatrixType::MaxColsAtCompileTime, +#ifndef EIGEN_TEST_EVALUATORS Flags = (unsigned int)ThenMatrixType::Flags & ElseMatrixType::Flags & HereditaryBits, CoeffReadCost = traits<typename remove_all<ConditionMatrixNested>::type>::CoeffReadCost + EIGEN_SIZE_MAX(traits<typename remove_all<ThenMatrixNested>::type>::CoeffReadCost, traits<typename remove_all<ElseMatrixNested>::type>::CoeffReadCost) +#else + Flags = (unsigned int)ThenMatrixType::Flags & ElseMatrixType::Flags & RowMajorBit +#endif }; }; } diff --git a/Eigen/src/Core/SelfAdjointView.h b/Eigen/src/Core/SelfAdjointView.h index 6c2733650..546f61252 100644 --- a/Eigen/src/Core/SelfAdjointView.h +++ b/Eigen/src/Core/SelfAdjointView.h @@ -35,26 +35,32 @@ struct traits<SelfAdjointView<MatrixType, UpLo> > : traits<MatrixType> typedef typename nested<MatrixType>::type MatrixTypeNested; typedef typename remove_all<MatrixTypeNested>::type MatrixTypeNestedCleaned; typedef MatrixType ExpressionType; - typedef typename MatrixType::PlainObject DenseMatrixType; + typedef typename MatrixType::PlainObject FullMatrixType; enum { Mode = UpLo | SelfAdjoint, Flags = MatrixTypeNestedCleaned::Flags & (HereditaryBits) - & (~(PacketAccessBit | DirectAccessBit | LinearAccessBit)), // FIXME these flags should be preserved + & (~(PacketAccessBit | DirectAccessBit | LinearAccessBit)) // FIXME these flags should be preserved +#ifndef EIGEN_TEST_EVALUATORS + , CoeffReadCost = MatrixTypeNestedCleaned::CoeffReadCost +#endif }; }; } +#ifndef EIGEN_TEST_EVALUATORS template <typename Lhs, int LhsMode, bool LhsIsVector, typename Rhs, int RhsMode, bool RhsIsVector> struct SelfadjointProductMatrix; +#endif // FIXME could also be called SelfAdjointWrapper to be consistent with DiagonalWrapper ?? -template<typename MatrixType, unsigned int UpLo> class SelfAdjointView - : public TriangularBase<SelfAdjointView<MatrixType, UpLo> > +template<typename _MatrixType, unsigned int UpLo> class SelfAdjointView + : public TriangularBase<SelfAdjointView<_MatrixType, UpLo> > { public: + typedef _MatrixType MatrixType; typedef TriangularBase<SelfAdjointView> Base; typedef typename internal::traits<SelfAdjointView>::MatrixTypeNested MatrixTypeNested; typedef typename internal::traits<SelfAdjointView>::MatrixTypeNestedCleaned MatrixTypeNestedCleaned; @@ -65,7 +71,8 @@ template<typename MatrixType, unsigned int UpLo> class SelfAdjointView typedef typename MatrixType::Index Index; enum { - Mode = internal::traits<SelfAdjointView>::Mode + Mode = internal::traits<SelfAdjointView>::Mode, + Flags = internal::traits<SelfAdjointView>::Flags }; typedef typename MatrixType::PlainObject PlainObject; @@ -111,6 +118,35 @@ template<typename MatrixType, unsigned int UpLo> class SelfAdjointView EIGEN_DEVICE_FUNC MatrixTypeNestedCleaned& nestedExpression() { return *const_cast<MatrixTypeNestedCleaned*>(&m_matrix); } +#ifdef EIGEN_TEST_EVALUATORS + + /** Efficient triangular matrix times vector/matrix product */ + template<typename OtherDerived> + EIGEN_DEVICE_FUNC + const Product<SelfAdjointView,OtherDerived> + operator*(const MatrixBase<OtherDerived>& rhs) const + { + return Product<SelfAdjointView,OtherDerived>(*this, rhs.derived()); + } + + /** Efficient vector/matrix times triangular matrix product */ + template<typename OtherDerived> friend + EIGEN_DEVICE_FUNC + const Product<OtherDerived,SelfAdjointView> + operator*(const MatrixBase<OtherDerived>& lhs, const SelfAdjointView& rhs) + { + return Product<OtherDerived,SelfAdjointView>(lhs.derived(),rhs); + } + + friend EIGEN_DEVICE_FUNC + const SelfAdjointView<const CwiseUnaryOp<internal::scalar_multiple_op<Scalar>,MatrixType>,UpLo> + operator*(const Scalar& s, const SelfAdjointView& mat) + { + return (s*mat.nestedExpression()).template selfadjointView<UpLo>(); + } + +#else // EIGEN_TEST_EVALUATORS + /** Efficient self-adjoint matrix times vector/matrix product */ template<typename OtherDerived> EIGEN_DEVICE_FUNC @@ -132,6 +168,7 @@ template<typename MatrixType, unsigned int UpLo> class SelfAdjointView <OtherDerived,0,OtherDerived::IsVectorAtCompileTime,MatrixType,Mode,false> (lhs.derived(),rhs.m_matrix); } +#endif /** Perform a symmetric rank 2 update of the selfadjoint matrix \c *this: * \f$ this = this + \alpha u v^* + conj(\alpha) v u^* \f$ @@ -194,6 +231,8 @@ template<typename MatrixType, unsigned int UpLo> class SelfAdjointView namespace internal { +#ifndef EIGEN_TEST_EVALUATORS + template<typename Derived1, typename Derived2, int UnrollCount, bool ClearOpposite> struct triangular_assignment_selector<Derived1, Derived2, (SelfAdjoint|Upper), UnrollCount, ClearOpposite> { @@ -286,6 +325,65 @@ struct triangular_assignment_selector<Derived1, Derived2, SelfAdjoint|Lower, Dyn } }; +#endif // EIGEN_TEST_EVALUATORS + +#ifdef EIGEN_ENABLE_EVALUATORS + +// TODO currently a selfadjoint expression has the form SelfAdjointView<.,.> +// in the future selfadjoint-ness should be defined by the expression traits +// such that Transpose<SelfAdjointView<.,.> > is valid. (currently TriangularBase::transpose() is overloaded to make it work) +template<typename MatrixType, unsigned int Mode> +struct evaluator_traits<SelfAdjointView<MatrixType,Mode> > +{ + typedef typename storage_kind_to_evaluator_kind<typename MatrixType::StorageKind>::Kind Kind; + typedef SelfAdjointShape Shape; + + static const int AssumeAliasing = 0; +}; + +template<int UpLo, int SetOpposite, typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT, typename Functor, int Version> +class triangular_dense_assignment_kernel<UpLo,SelfAdjoint,SetOpposite,DstEvaluatorTypeT,SrcEvaluatorTypeT,Functor,Version> + : public generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, Version> +{ +protected: + typedef generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, Version> Base; + typedef typename Base::DstXprType DstXprType; + typedef typename Base::SrcXprType SrcXprType; + using Base::m_dst; + using Base::m_src; + using Base::m_functor; +public: + + typedef typename Base::DstEvaluatorType DstEvaluatorType; + typedef typename Base::SrcEvaluatorType SrcEvaluatorType; + typedef typename Base::Scalar Scalar; + typedef typename Base::Index Index; + typedef typename Base::AssignmentTraits AssignmentTraits; + + + triangular_dense_assignment_kernel(DstEvaluatorType &dst, const SrcEvaluatorType &src, const Functor &func, DstXprType& dstExpr) + : Base(dst, src, func, dstExpr) + {} + + void assignCoeff(Index row, Index col) + { + eigen_internal_assert(row!=col); + Scalar tmp = m_src.coeff(row,col); + m_functor.assignCoeff(m_dst.coeffRef(row,col), tmp); + m_functor.assignCoeff(m_dst.coeffRef(col,row), numext::conj(tmp)); + } + + void assignDiagonalCoeff(Index id) + { + Base::assignCoeff(id,id); + } + + void assignOppositeCoeff(Index, Index) + { eigen_internal_assert(false && "should never be called"); } +}; + +#endif // EIGEN_ENABLE_EVALUATORS + } // end namespace internal /*************************************************************************** diff --git a/Eigen/src/Core/SelfCwiseBinaryOp.h b/Eigen/src/Core/SelfCwiseBinaryOp.h index 65864adf8..ae7f9b887 100644 --- a/Eigen/src/Core/SelfCwiseBinaryOp.h +++ b/Eigen/src/Core/SelfCwiseBinaryOp.h @@ -12,6 +12,8 @@ namespace Eigen { +#ifndef EIGEN_TEST_EVALUATORS + /** \class SelfCwiseBinaryOp * \ingroup Core_Module * @@ -179,6 +181,51 @@ template<typename BinaryOp, typename Lhs, typename Rhs> class SelfCwiseBinaryOp SelfCwiseBinaryOp& operator=(const SelfCwiseBinaryOp&); }; +#endif // EIGEN_TEST_EVALUATORS + +#ifdef EIGEN_TEST_EVALUATORS +template<typename Derived> +inline Derived& DenseBase<Derived>::operator*=(const Scalar& other) +{ + typedef typename Derived::PlainObject PlainObject; + internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::mul_assign_op<Scalar>()); + return derived(); +} + +template<typename Derived> +inline Derived& ArrayBase<Derived>::operator+=(const Scalar& other) +{ + typedef typename Derived::PlainObject PlainObject; + internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::add_assign_op<Scalar>()); + return derived(); +} + +template<typename Derived> +inline Derived& ArrayBase<Derived>::operator-=(const Scalar& other) +{ + typedef typename Derived::PlainObject PlainObject; + internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),other), internal::sub_assign_op<Scalar>()); + return derived(); +} + +template<typename Derived> +inline Derived& DenseBase<Derived>::operator/=(const Scalar& other) +{ + typedef typename Derived::PlainObject PlainObject; + + typedef typename internal::conditional<NumTraits<Scalar>::IsInteger, + internal::div_assign_op<Scalar>, + internal::mul_assign_op<Scalar> >::type AssignOp; + + Scalar actual_other; + if(NumTraits<Scalar>::IsInteger) actual_other = other; + else actual_other = Scalar(1)/other; + + internal::call_assignment(this->derived(), PlainObject::Constant(rows(),cols(),actual_other), AssignOp()); + + return derived(); +} +#else template<typename Derived> inline Derived& DenseBase<Derived>::operator*=(const Scalar& other) { @@ -220,6 +267,7 @@ inline Derived& DenseBase<Derived>::operator/=(const Scalar& other) tmp = PlainObject::Constant(rows(),cols(), actual_other); return derived(); } +#endif } // end namespace Eigen diff --git a/Eigen/src/Core/Solve.h b/Eigen/src/Core/Solve.h new file mode 100644 index 000000000..c28789968 --- /dev/null +++ b/Eigen/src/Core/Solve.h @@ -0,0 +1,145 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_SOLVE_H +#define EIGEN_SOLVE_H + +namespace Eigen { + +template<typename Decomposition, typename RhsType, typename StorageKind> class SolveImpl; + +/** \class Solve + * \ingroup Core_Module + * + * \brief Pseudo expression representing a solving operation + * + * \tparam Decomposition the type of the matrix or decomposion object + * \tparam Rhstype the type of the right-hand side + * + * This class represents an expression of A.solve(B) + * and most of the time this is the only way it is used. + * + */ +namespace internal { + +// this solve_traits class permits to determine the evaluation type with respect to storage kind (Dense vs Sparse) +template<typename Decomposition, typename RhsType,typename StorageKind> struct solve_traits; + +template<typename Decomposition, typename RhsType> +struct solve_traits<Decomposition,RhsType,Dense> +{ + typedef typename Decomposition::MatrixType MatrixType; + typedef Matrix<typename RhsType::Scalar, + MatrixType::ColsAtCompileTime, + RhsType::ColsAtCompileTime, + RhsType::PlainObject::Options, + MatrixType::MaxColsAtCompileTime, + RhsType::MaxColsAtCompileTime> PlainObject; +}; + +template<typename Decomposition, typename RhsType> +struct traits<Solve<Decomposition, RhsType> > + : traits<typename solve_traits<Decomposition,RhsType,typename internal::traits<RhsType>::StorageKind>::PlainObject> +{ + typedef typename solve_traits<Decomposition,RhsType,typename internal::traits<RhsType>::StorageKind>::PlainObject PlainObject; + typedef traits<PlainObject> BaseTraits; + enum { + Flags = BaseTraits::Flags & RowMajorBit, + CoeffReadCost = Dynamic + }; +}; + +} + + +template<typename Decomposition, typename RhsType> +class Solve : public SolveImpl<Decomposition,RhsType,typename internal::traits<RhsType>::StorageKind> +{ +public: + typedef typename RhsType::Index Index; + typedef typename internal::traits<Solve>::PlainObject PlainObject; + + Solve(const Decomposition &dec, const RhsType &rhs) + : m_dec(dec), m_rhs(rhs) + {} + + EIGEN_DEVICE_FUNC Index rows() const { return m_dec.cols(); } + EIGEN_DEVICE_FUNC Index cols() const { return m_rhs.cols(); } + + EIGEN_DEVICE_FUNC const Decomposition& dec() const { return m_dec; } + EIGEN_DEVICE_FUNC const RhsType& rhs() const { return m_rhs; } + +protected: + const Decomposition &m_dec; + const RhsType &m_rhs; +}; + + +// Specialization of the Solve expression for dense results +template<typename Decomposition, typename RhsType> +class SolveImpl<Decomposition,RhsType,Dense> + : public MatrixBase<Solve<Decomposition,RhsType> > +{ + typedef Solve<Decomposition,RhsType> Derived; + +public: + + typedef MatrixBase<Solve<Decomposition,RhsType> > Base; + EIGEN_DENSE_PUBLIC_INTERFACE(Derived) + +private: + + Scalar coeff(Index row, Index col) const; + Scalar coeff(Index i) const; +}; + + +namespace internal { + +// Evaluator of Solve -> eval into a temporary +template<typename Decomposition, typename RhsType> +struct evaluator<Solve<Decomposition,RhsType> > + : public evaluator<typename Solve<Decomposition,RhsType>::PlainObject>::type +{ + typedef Solve<Decomposition,RhsType> SolveType; + typedef typename SolveType::PlainObject PlainObject; + typedef typename evaluator<PlainObject>::type Base; + + typedef evaluator type; + typedef evaluator nestedType; + + evaluator(const SolveType& solve) + : m_result(solve.rows(), solve.cols()) + { + ::new (static_cast<Base*>(this)) Base(m_result); + solve.dec()._solve_impl(solve.rhs(), m_result); + } + +protected: + PlainObject m_result; +}; + +// Specialization for "dst = dec.solve(rhs)" +// NOTE we need to specialize it for Dense2Dense to avoid ambiguous specialization error and a Sparse2Sparse specialization must exist somewhere +template<typename DstXprType, typename DecType, typename RhsType, typename Scalar> +struct Assignment<DstXprType, Solve<DecType,RhsType>, internal::assign_op<Scalar>, Dense2Dense, Scalar> +{ + typedef Solve<DecType,RhsType> SrcXprType; + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + { + // FIXME shall we resize dst here? + src.dec()._solve_impl(src.rhs(), dst); + } +}; + +} // end namepsace internal + +} // end namespace Eigen + +#endif // EIGEN_SOLVE_H diff --git a/Eigen/src/Core/SolveTriangular.h b/Eigen/src/Core/SolveTriangular.h index ef17f288e..0f17e3a89 100644 --- a/Eigen/src/Core/SolveTriangular.h +++ b/Eigen/src/Core/SolveTriangular.h @@ -171,10 +171,10 @@ struct triangular_solver_selector<Lhs,Rhs,OnTheRight,Mode,CompleteUnrolling,1> { */ template<typename MatrixType, unsigned int Mode> template<int Side, typename OtherDerived> -void TriangularView<MatrixType,Mode>::solveInPlace(const MatrixBase<OtherDerived>& _other) const +void TriangularViewImpl<MatrixType,Mode,Dense>::solveInPlace(const MatrixBase<OtherDerived>& _other) const { OtherDerived& other = _other.const_cast_derived(); - eigen_assert( cols() == rows() && ((Side==OnTheLeft && cols() == other.rows()) || (Side==OnTheRight && cols() == other.cols())) ); + eigen_assert( derived().cols() == derived().rows() && ((Side==OnTheLeft && derived().cols() == other.rows()) || (Side==OnTheRight && derived().cols() == other.cols())) ); eigen_assert((!(Mode & ZeroDiag)) && bool(Mode & (Upper|Lower))); enum { copy = internal::traits<OtherDerived>::Flags & RowMajorBit && OtherDerived::IsVectorAtCompileTime }; @@ -183,7 +183,7 @@ void TriangularView<MatrixType,Mode>::solveInPlace(const MatrixBase<OtherDerived OtherCopy otherCopy(other); internal::triangular_solver_selector<MatrixType, typename internal::remove_reference<OtherCopy>::type, - Side, Mode>::run(nestedExpression(), otherCopy); + Side, Mode>::run(derived().nestedExpression(), otherCopy); if (copy) other = otherCopy; @@ -213,9 +213,9 @@ void TriangularView<MatrixType,Mode>::solveInPlace(const MatrixBase<OtherDerived template<typename Derived, unsigned int Mode> template<int Side, typename Other> const internal::triangular_solve_retval<Side,TriangularView<Derived,Mode>,Other> -TriangularView<Derived,Mode>::solve(const MatrixBase<Other>& other) const +TriangularViewImpl<Derived,Mode,Dense>::solve(const MatrixBase<Other>& other) const { - return internal::triangular_solve_retval<Side,TriangularView,Other>(*this, other.derived()); + return internal::triangular_solve_retval<Side,TriangularViewType,Other>(derived(), other.derived()); } namespace internal { diff --git a/Eigen/src/Core/Stride.h b/Eigen/src/Core/Stride.h index d3d454e4e..187774978 100644 --- a/Eigen/src/Core/Stride.h +++ b/Eigen/src/Core/Stride.h @@ -86,7 +86,7 @@ class Stride /** \brief Convenience specialization of Stride to specify only an inner stride * See class Map for some examples */ -template<int Value = Dynamic> +template<int Value> class InnerStride : public Stride<0, Value> { typedef Stride<0, Value> Base; @@ -98,7 +98,7 @@ class InnerStride : public Stride<0, Value> /** \brief Convenience specialization of Stride to specify only an outer stride * See class Map for some examples */ -template<int Value = Dynamic> +template<int Value> class OuterStride : public Stride<Value, 0> { typedef Stride<Value, 0> Base; diff --git a/Eigen/src/Core/Swap.h b/Eigen/src/Core/Swap.h index d602fba65..9a1c5f4f8 100644 --- a/Eigen/src/Core/Swap.h +++ b/Eigen/src/Core/Swap.h @@ -12,6 +12,8 @@ namespace Eigen { +#ifndef EIGEN_TEST_EVALUATORS + /** \class SwapWrapper * \ingroup Core_Module * @@ -135,6 +137,60 @@ template<typename ExpressionType> class SwapWrapper ExpressionType& m_expression; }; +#endif + +#ifdef EIGEN_ENABLE_EVALUATORS + +namespace internal { + +// Overload default assignPacket behavior for swapping them +template<typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT> +class generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, swap_assign_op<typename DstEvaluatorTypeT::Scalar>, Specialized> + : public generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, swap_assign_op<typename DstEvaluatorTypeT::Scalar>, BuiltIn> +{ +protected: + typedef generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, swap_assign_op<typename DstEvaluatorTypeT::Scalar>, BuiltIn> Base; + typedef typename DstEvaluatorTypeT::PacketScalar PacketScalar; + using Base::m_dst; + using Base::m_src; + using Base::m_functor; + +public: + typedef typename Base::Scalar Scalar; + typedef typename Base::Index Index; + typedef typename Base::DstXprType DstXprType; + typedef swap_assign_op<Scalar> Functor; + + generic_dense_assignment_kernel(DstEvaluatorTypeT &dst, const SrcEvaluatorTypeT &src, const Functor &func, DstXprType& dstExpr) + : Base(dst, src, func, dstExpr) + {} + + template<int StoreMode, int LoadMode> + void assignPacket(Index row, Index col) + { + m_functor.template swapPacket<StoreMode,LoadMode,PacketScalar>(&m_dst.coeffRef(row,col), &const_cast<SrcEvaluatorTypeT&>(m_src).coeffRef(row,col)); + } + + template<int StoreMode, int LoadMode> + void assignPacket(Index index) + { + m_functor.template swapPacket<StoreMode,LoadMode,PacketScalar>(&m_dst.coeffRef(index), &const_cast<SrcEvaluatorTypeT&>(m_src).coeffRef(index)); + } + + // TODO find a simple way not to have to copy/paste this function from generic_dense_assignment_kernel, by simple I mean no CRTP (Gael) + template<int StoreMode, int LoadMode> + void assignPacketByOuterInner(Index outer, Index inner) + { + Index row = Base::rowIndexByOuterInner(outer, inner); + Index col = Base::colIndexByOuterInner(outer, inner); + assignPacket<StoreMode,LoadMode>(row, col); + } +}; + +} // namespace internal + +#endif + } // end namespace Eigen #endif // EIGEN_SWAP_H diff --git a/Eigen/src/Core/Transpose.h b/Eigen/src/Core/Transpose.h index aba3f6670..f5148221d 100644 --- a/Eigen/src/Core/Transpose.h +++ b/Eigen/src/Core/Transpose.h @@ -2,7 +2,7 @@ // for linear algebra. // // Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com> -// Copyright (C) 2009-2010 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2009-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -41,11 +41,18 @@ struct traits<Transpose<MatrixType> > : traits<MatrixType> ColsAtCompileTime = MatrixType::RowsAtCompileTime, MaxRowsAtCompileTime = MatrixType::MaxColsAtCompileTime, MaxColsAtCompileTime = MatrixType::MaxRowsAtCompileTime, +#ifndef EIGEN_TEST_EVALUATORS FlagsLvalueBit = is_lvalue<MatrixType>::value ? LvalueBit : 0, Flags0 = MatrixTypeNestedPlain::Flags & ~(LvalueBit | NestByRefBit), Flags1 = Flags0 | FlagsLvalueBit, Flags = Flags1 ^ RowMajorBit, CoeffReadCost = MatrixTypeNestedPlain::CoeffReadCost, +#else + FlagsLvalueBit = is_lvalue<MatrixType>::value ? LvalueBit : 0, + Flags0 = MatrixTypeNestedPlain::Flags & ~(LvalueBit | NestByRefBit), + Flags1 = Flags0 | FlagsLvalueBit, + Flags = Flags1 ^ RowMajorBit, +#endif InnerStrideAtCompileTime = inner_stride_at_compile_time<MatrixType>::ret, OuterStrideAtCompileTime = outer_stride_at_compile_time<MatrixType>::ret }; @@ -61,6 +68,7 @@ template<typename MatrixType> class Transpose typedef typename TransposeImpl<MatrixType,typename internal::traits<MatrixType>::StorageKind>::Base Base; EIGEN_GENERIC_PUBLIC_INTERFACE(Transpose) + typedef typename internal::remove_all<MatrixType>::type NestedExpression; EIGEN_DEVICE_FUNC inline Transpose(MatrixType& a_matrix) : m_matrix(a_matrix) {} @@ -100,12 +108,24 @@ struct TransposeImpl_base<MatrixType, false> } // end namespace internal +#ifdef EIGEN_TEST_EVALUATORS +// Generic API dispatcher +template<typename XprType, typename StorageKind> +class TransposeImpl + : public internal::generic_xpr_base<Transpose<XprType> >::type +{ +public: + typedef typename internal::generic_xpr_base<Transpose<XprType> >::type Base; +}; +#endif + template<typename MatrixType> class TransposeImpl<MatrixType,Dense> : public internal::TransposeImpl_base<MatrixType>::type { public: typedef typename internal::TransposeImpl_base<MatrixType>::type Base; + using Base::coeffRef; EIGEN_DENSE_PUBLIC_INTERFACE(Transpose<MatrixType>) EIGEN_INHERIT_ASSIGNMENT_OPERATORS(TransposeImpl) @@ -120,6 +140,8 @@ template<typename MatrixType> class TransposeImpl<MatrixType,Dense> inline ScalarWithConstIfNotLvalue* data() { return derived().nestedExpression().data(); } inline const Scalar* data() const { return derived().nestedExpression().data(); } + +#ifndef EIGEN_TEST_EVALUATORS EIGEN_DEVICE_FUNC inline ScalarWithConstIfNotLvalue& coeffRef(Index rowId, Index colId) @@ -136,18 +158,6 @@ template<typename MatrixType> class TransposeImpl<MatrixType,Dense> } EIGEN_DEVICE_FUNC - inline const Scalar& coeffRef(Index rowId, Index colId) const - { - return derived().nestedExpression().coeffRef(colId, rowId); - } - - EIGEN_DEVICE_FUNC - inline const Scalar& coeffRef(Index index) const - { - return derived().nestedExpression().coeffRef(index); - } - - EIGEN_DEVICE_FUNC inline CoeffReturnType coeff(Index rowId, Index colId) const { return derived().nestedExpression().coeff(colId, rowId); @@ -182,6 +192,20 @@ template<typename MatrixType> class TransposeImpl<MatrixType,Dense> { derived().nestedExpression().const_cast_derived().template writePacket<LoadMode>(index, x); } +#endif + + // FIXME: shall we keep the const version of coeffRef? + EIGEN_DEVICE_FUNC + inline const Scalar& coeffRef(Index rowId, Index colId) const + { + return derived().nestedExpression().coeffRef(colId, rowId); + } + + EIGEN_DEVICE_FUNC + inline const Scalar& coeffRef(Index index) const + { + return derived().nestedExpression().coeffRef(index); + } }; /** \returns an expression of the transpose of *this. diff --git a/Eigen/src/Core/TriangularMatrix.h b/Eigen/src/Core/TriangularMatrix.h index 72792d21b..0383ca9f5 100644 --- a/Eigen/src/Core/TriangularMatrix.h +++ b/Eigen/src/Core/TriangularMatrix.h @@ -32,17 +32,26 @@ template<typename Derived> class TriangularBase : public EigenBase<Derived> enum { Mode = internal::traits<Derived>::Mode, +#ifndef EIGEN_TEST_EVALUATORS CoeffReadCost = internal::traits<Derived>::CoeffReadCost, +#endif RowsAtCompileTime = internal::traits<Derived>::RowsAtCompileTime, ColsAtCompileTime = internal::traits<Derived>::ColsAtCompileTime, MaxRowsAtCompileTime = internal::traits<Derived>::MaxRowsAtCompileTime, - MaxColsAtCompileTime = internal::traits<Derived>::MaxColsAtCompileTime + MaxColsAtCompileTime = internal::traits<Derived>::MaxColsAtCompileTime, + + SizeAtCompileTime = (internal::size_at_compile_time<internal::traits<Derived>::RowsAtCompileTime, + internal::traits<Derived>::ColsAtCompileTime>::ret) + /**< This is equal to the number of coefficients, i.e. the number of + * rows times the number of columns, or to \a Dynamic if this is not + * known at compile-time. \sa RowsAtCompileTime, ColsAtCompileTime */ }; typedef typename internal::traits<Derived>::Scalar Scalar; typedef typename internal::traits<Derived>::StorageKind StorageKind; typedef typename internal::traits<Derived>::Index Index; - typedef typename internal::traits<Derived>::DenseMatrixType DenseMatrixType; + typedef typename internal::traits<Derived>::FullMatrixType DenseMatrixType; typedef DenseMatrixType DenseType; + typedef Derived const& Nested; EIGEN_DEVICE_FUNC inline TriangularBase() { eigen_assert(!((Mode&UnitDiag) && (Mode&ZeroDiag))); } @@ -55,6 +64,14 @@ template<typename Derived> class TriangularBase : public EigenBase<Derived> inline Index outerStride() const { return derived().outerStride(); } EIGEN_DEVICE_FUNC inline Index innerStride() const { return derived().innerStride(); } + + // dummy resize function + void resize(Index nbRows, Index nbCols) + { + EIGEN_UNUSED_VARIABLE(nbRows); + EIGEN_UNUSED_VARIABLE(nbCols); + eigen_assert(nbRows==rows() && nbCols==nbCols); + } EIGEN_DEVICE_FUNC inline Scalar coeff(Index row, Index col) const { return derived().coeff(row,col); } @@ -155,49 +172,52 @@ struct traits<TriangularView<MatrixType, _Mode> > : traits<MatrixType> typedef typename nested<MatrixType>::type MatrixTypeNested; typedef typename remove_reference<MatrixTypeNested>::type MatrixTypeNestedNonRef; typedef typename remove_all<MatrixTypeNested>::type MatrixTypeNestedCleaned; + typedef typename MatrixType::PlainObject FullMatrixType; typedef MatrixType ExpressionType; - typedef typename MatrixType::PlainObject DenseMatrixType; enum { Mode = _Mode, - Flags = (MatrixTypeNestedCleaned::Flags & (HereditaryBits) & (~(PacketAccessBit | DirectAccessBit | LinearAccessBit))) | Mode, + Flags = (MatrixTypeNestedCleaned::Flags & (HereditaryBits | LvalueBit) & (~(PacketAccessBit | DirectAccessBit | LinearAccessBit))) +#ifndef EIGEN_TEST_EVALUATORS + , CoeffReadCost = MatrixTypeNestedCleaned::CoeffReadCost +#endif }; }; } +#ifndef EIGEN_TEST_EVALUATORS template<int Mode, bool LhsIsTriangular, typename Lhs, bool LhsIsVector, typename Rhs, bool RhsIsVector> struct TriangularProduct; +#endif + +template<typename _MatrixType, unsigned int _Mode, typename StorageKind> class TriangularViewImpl; template<typename _MatrixType, unsigned int _Mode> class TriangularView - : public TriangularBase<TriangularView<_MatrixType, _Mode> > + : public TriangularViewImpl<_MatrixType, _Mode, typename internal::traits<_MatrixType>::StorageKind > { public: - typedef TriangularBase<TriangularView> Base; + typedef TriangularViewImpl<_MatrixType, _Mode, typename internal::traits<_MatrixType>::StorageKind > Base; typedef typename internal::traits<TriangularView>::Scalar Scalar; - typedef _MatrixType MatrixType; - typedef typename internal::traits<TriangularView>::DenseMatrixType DenseMatrixType; - typedef DenseMatrixType PlainObject; protected: typedef typename internal::traits<TriangularView>::MatrixTypeNested MatrixTypeNested; typedef typename internal::traits<TriangularView>::MatrixTypeNestedNonRef MatrixTypeNestedNonRef; - typedef typename internal::traits<TriangularView>::MatrixTypeNestedCleaned MatrixTypeNestedCleaned; typedef typename internal::remove_all<typename MatrixType::ConjugateReturnType>::type MatrixConjugateReturnType; public: - using Base::evalToLazy; - typedef typename internal::traits<TriangularView>::StorageKind StorageKind; typedef typename internal::traits<TriangularView>::Index Index; + typedef typename internal::traits<TriangularView>::MatrixTypeNestedCleaned NestedExpression; enum { Mode = _Mode, + Flags = internal::traits<TriangularView>::Flags, TransposeMode = (Mode & Upper ? Lower : 0) | (Mode & Lower ? Upper : 0) | (Mode & (UnitDiag)) @@ -207,44 +227,165 @@ template<typename _MatrixType, unsigned int _Mode> class TriangularView EIGEN_DEVICE_FUNC inline TriangularView(const MatrixType& matrix) : m_matrix(matrix) {} + + using Base::operator=; + TriangularView& operator=(const TriangularView &other) + { return Base::operator=(other); } EIGEN_DEVICE_FUNC inline Index rows() const { return m_matrix.rows(); } EIGEN_DEVICE_FUNC inline Index cols() const { return m_matrix.cols(); } + + EIGEN_DEVICE_FUNC + const NestedExpression& nestedExpression() const { return m_matrix; } + EIGEN_DEVICE_FUNC + NestedExpression& nestedExpression() { return *const_cast<NestedExpression*>(&m_matrix); } + + /** \sa MatrixBase::conjugate() */ + EIGEN_DEVICE_FUNC + inline TriangularView<MatrixConjugateReturnType,Mode> conjugate() + { return m_matrix.conjugate(); } + /** \sa MatrixBase::conjugate() const */ + EIGEN_DEVICE_FUNC + inline const TriangularView<MatrixConjugateReturnType,Mode> conjugate() const + { return m_matrix.conjugate(); } + + /** \sa MatrixBase::adjoint() const */ + EIGEN_DEVICE_FUNC + inline const TriangularView<const typename MatrixType::AdjointReturnType,TransposeMode> adjoint() const + { return m_matrix.adjoint(); } + + /** \sa MatrixBase::transpose() */ + EIGEN_DEVICE_FUNC + inline TriangularView<Transpose<MatrixType>,TransposeMode> transpose() + { + EIGEN_STATIC_ASSERT_LVALUE(MatrixType) + return m_matrix.const_cast_derived().transpose(); + } + /** \sa MatrixBase::transpose() const */ + EIGEN_DEVICE_FUNC + inline const TriangularView<Transpose<MatrixType>,TransposeMode> transpose() const + { + return m_matrix.transpose(); + } + +#ifdef EIGEN_TEST_EVALUATORS + template<typename Other> + EIGEN_DEVICE_FUNC + inline const Solve<TriangularView, Other> + solve(const MatrixBase<Other>& other) const + { return Solve<TriangularView, Other>(*this, other.derived()); } + + using Base::solve; +#endif // EIGEN_TEST_EVALUATORS + + EIGEN_DEVICE_FUNC + const SelfAdjointView<MatrixTypeNestedNonRef,Mode> selfadjointView() const + { + EIGEN_STATIC_ASSERT((Mode&UnitDiag)==0,PROGRAMMING_ERROR); + return SelfAdjointView<MatrixTypeNestedNonRef,Mode>(m_matrix); + } + EIGEN_DEVICE_FUNC + SelfAdjointView<MatrixTypeNestedNonRef,Mode> selfadjointView() + { + EIGEN_STATIC_ASSERT((Mode&UnitDiag)==0,PROGRAMMING_ERROR); + return SelfAdjointView<MatrixTypeNestedNonRef,Mode>(m_matrix); + } + + EIGEN_DEVICE_FUNC + Scalar determinant() const + { + if (Mode & UnitDiag) + return 1; + else if (Mode & ZeroDiag) + return 0; + else + return m_matrix.diagonal().prod(); + } + + protected: + + MatrixTypeNested m_matrix; +}; + +template<typename _MatrixType, unsigned int _Mode> class TriangularViewImpl<_MatrixType,_Mode,Dense> + : public TriangularBase<TriangularView<_MatrixType, _Mode> > +{ + public: + + typedef TriangularView<_MatrixType, _Mode> TriangularViewType; + typedef TriangularBase<TriangularViewType> Base; + typedef typename internal::traits<TriangularViewType>::Scalar Scalar; + + typedef _MatrixType MatrixType; + typedef typename MatrixType::PlainObject DenseMatrixType; + typedef DenseMatrixType PlainObject; + + public: + using Base::evalToLazy; + using Base::derived; + + typedef typename internal::traits<TriangularViewType>::StorageKind StorageKind; + typedef typename internal::traits<TriangularViewType>::Index Index; + + enum { + Mode = _Mode, + Flags = internal::traits<TriangularViewType>::Flags + }; + EIGEN_DEVICE_FUNC - inline Index outerStride() const { return m_matrix.outerStride(); } + inline Index outerStride() const { return derived().nestedExpression().outerStride(); } EIGEN_DEVICE_FUNC - inline Index innerStride() const { return m_matrix.innerStride(); } + inline Index innerStride() const { return derived().nestedExpression().innerStride(); } +#ifdef EIGEN_TEST_EVALUATORS + + /** \sa MatrixBase::operator+=() */ + template<typename Other> + EIGEN_DEVICE_FUNC + TriangularViewType& operator+=(const DenseBase<Other>& other) { + internal::call_assignment_no_alias(derived(), other.derived(), internal::add_assign_op<Scalar>()); + return derived(); + } + /** \sa MatrixBase::operator-=() */ + template<typename Other> + EIGEN_DEVICE_FUNC + TriangularViewType& operator-=(const DenseBase<Other>& other) { + internal::call_assignment_no_alias(derived(), other.derived(), internal::sub_assign_op<Scalar>()); + return derived(); + } + +#else /** \sa MatrixBase::operator+=() */ template<typename Other> EIGEN_DEVICE_FUNC - TriangularView& operator+=(const DenseBase<Other>& other) { return *this = m_matrix + other.derived(); } + TriangularViewType& operator+=(const DenseBase<Other>& other) { return *this = derived().nestedExpression() + other.derived(); } /** \sa MatrixBase::operator-=() */ template<typename Other> EIGEN_DEVICE_FUNC - TriangularView& operator-=(const DenseBase<Other>& other) { return *this = m_matrix - other.derived(); } + TriangularViewType& operator-=(const DenseBase<Other>& other) { return *this = derived().nestedExpression() - other.derived(); } +#endif /** \sa MatrixBase::operator*=() */ EIGEN_DEVICE_FUNC - TriangularView& operator*=(const typename internal::traits<MatrixType>::Scalar& other) { return *this = m_matrix * other; } + TriangularViewType& operator*=(const typename internal::traits<MatrixType>::Scalar& other) { return *this = derived().nestedExpression() * other; } /** \sa MatrixBase::operator/=() */ EIGEN_DEVICE_FUNC - TriangularView& operator/=(const typename internal::traits<MatrixType>::Scalar& other) { return *this = m_matrix / other; } + TriangularViewType& operator/=(const typename internal::traits<MatrixType>::Scalar& other) { return *this = derived().nestedExpression() / other; } /** \sa MatrixBase::fill() */ EIGEN_DEVICE_FUNC void fill(const Scalar& value) { setConstant(value); } /** \sa MatrixBase::setConstant() */ EIGEN_DEVICE_FUNC - TriangularView& setConstant(const Scalar& value) - { return *this = MatrixType::Constant(rows(), cols(), value); } + TriangularViewType& setConstant(const Scalar& value) + { return *this = MatrixType::Constant(derived().rows(), derived().cols(), value); } /** \sa MatrixBase::setZero() */ EIGEN_DEVICE_FUNC - TriangularView& setZero() { return setConstant(Scalar(0)); } + TriangularViewType& setZero() { return setConstant(Scalar(0)); } /** \sa MatrixBase::setOnes() */ EIGEN_DEVICE_FUNC - TriangularView& setOnes() { return setConstant(Scalar(1)); } + TriangularViewType& setOnes() { return setConstant(Scalar(1)); } /** \sa MatrixBase::coeff() * \warning the coordinates must fit into the referenced triangular part @@ -253,7 +394,7 @@ template<typename _MatrixType, unsigned int _Mode> class TriangularView inline Scalar coeff(Index row, Index col) const { Base::check_coordinates_internal(row, col); - return m_matrix.coeff(row, col); + return derived().nestedExpression().coeff(row, col); } /** \sa MatrixBase::coeffRef() @@ -263,26 +404,21 @@ template<typename _MatrixType, unsigned int _Mode> class TriangularView inline Scalar& coeffRef(Index row, Index col) { Base::check_coordinates_internal(row, col); - return m_matrix.const_cast_derived().coeffRef(row, col); + return derived().nestedExpression().const_cast_derived().coeffRef(row, col); } - EIGEN_DEVICE_FUNC - const MatrixTypeNestedCleaned& nestedExpression() const { return m_matrix; } - EIGEN_DEVICE_FUNC - MatrixTypeNestedCleaned& nestedExpression() { return *const_cast<MatrixTypeNestedCleaned*>(&m_matrix); } - /** Assigns a triangular matrix to a triangular part of a dense matrix */ template<typename OtherDerived> EIGEN_DEVICE_FUNC - TriangularView& operator=(const TriangularBase<OtherDerived>& other); + TriangularViewType& operator=(const TriangularBase<OtherDerived>& other); template<typename OtherDerived> EIGEN_DEVICE_FUNC - TriangularView& operator=(const MatrixBase<OtherDerived>& other); + TriangularViewType& operator=(const MatrixBase<OtherDerived>& other); EIGEN_DEVICE_FUNC - TriangularView& operator=(const TriangularView& other) - { return *this = other.nestedExpression(); } + TriangularViewType& operator=(const TriangularViewImpl& other) + { return *this = other.derived().nestedExpression(); } template<typename OtherDerived> EIGEN_DEVICE_FUNC @@ -290,36 +426,29 @@ template<typename _MatrixType, unsigned int _Mode> class TriangularView template<typename OtherDerived> EIGEN_DEVICE_FUNC - void lazyAssign(const MatrixBase<OtherDerived>& other); + void lazyAssign(const MatrixBase<OtherDerived>& other); - /** \sa MatrixBase::conjugate() */ - EIGEN_DEVICE_FUNC - inline TriangularView<MatrixConjugateReturnType,Mode> conjugate() - { return m_matrix.conjugate(); } - /** \sa MatrixBase::conjugate() const */ - EIGEN_DEVICE_FUNC - inline const TriangularView<MatrixConjugateReturnType,Mode> conjugate() const - { return m_matrix.conjugate(); } +#ifdef EIGEN_TEST_EVALUATORS - /** \sa MatrixBase::adjoint() const */ - EIGEN_DEVICE_FUNC - inline const TriangularView<const typename MatrixType::AdjointReturnType,TransposeMode> adjoint() const - { return m_matrix.adjoint(); } - - /** \sa MatrixBase::transpose() */ + /** Efficient triangular matrix times vector/matrix product */ + template<typename OtherDerived> EIGEN_DEVICE_FUNC - inline TriangularView<Transpose<MatrixType>,TransposeMode> transpose() + const Product<TriangularViewType,OtherDerived> + operator*(const MatrixBase<OtherDerived>& rhs) const { - EIGEN_STATIC_ASSERT_LVALUE(MatrixType) - return m_matrix.const_cast_derived().transpose(); + return Product<TriangularViewType,OtherDerived>(derived(), rhs.derived()); } - /** \sa MatrixBase::transpose() const */ + + /** Efficient vector/matrix times triangular matrix product */ + template<typename OtherDerived> friend EIGEN_DEVICE_FUNC - inline const TriangularView<Transpose<MatrixType>,TransposeMode> transpose() const + const Product<OtherDerived,TriangularViewType> + operator*(const MatrixBase<OtherDerived>& lhs, const TriangularViewImpl& rhs) { - return m_matrix.transpose(); + return Product<OtherDerived,TriangularViewType>(lhs.derived(),rhs.derived()); } - + +#else // EIGEN_TEST_EVALUATORS /** Efficient triangular matrix times vector/matrix product */ template<typename OtherDerived> EIGEN_DEVICE_FUNC @@ -328,83 +457,73 @@ template<typename _MatrixType, unsigned int _Mode> class TriangularView { return TriangularProduct <Mode, true, MatrixType, false, OtherDerived, OtherDerived::ColsAtCompileTime==1> - (m_matrix, rhs.derived()); + (derived().nestedExpression(), rhs.derived()); } /** Efficient vector/matrix times triangular matrix product */ template<typename OtherDerived> friend EIGEN_DEVICE_FUNC TriangularProduct<Mode, false, OtherDerived, OtherDerived::RowsAtCompileTime==1, MatrixType, false> - operator*(const MatrixBase<OtherDerived>& lhs, const TriangularView& rhs) + operator*(const MatrixBase<OtherDerived>& lhs, const TriangularViewImpl& rhs) { return TriangularProduct <Mode, false, OtherDerived, OtherDerived::RowsAtCompileTime==1, MatrixType, false> - (lhs.derived(),rhs.m_matrix); + (lhs.derived(),rhs.derived().nestedExpression()); } +#endif template<int Side, typename Other> EIGEN_DEVICE_FUNC - inline const internal::triangular_solve_retval<Side,TriangularView, Other> + inline const internal::triangular_solve_retval<Side,TriangularViewType, Other> solve(const MatrixBase<Other>& other) const; template<int Side, typename OtherDerived> EIGEN_DEVICE_FUNC void solveInPlace(const MatrixBase<OtherDerived>& other) const; +#ifndef EIGEN_TEST_EVALUATORS template<typename Other> EIGEN_DEVICE_FUNC - inline const internal::triangular_solve_retval<OnTheLeft,TriangularView, Other> + inline const internal::triangular_solve_retval<OnTheLeft,TriangularViewType, Other> solve(const MatrixBase<Other>& other) const { return solve<OnTheLeft>(other); } +#endif // EIGEN_TEST_EVALUATORS template<typename OtherDerived> EIGEN_DEVICE_FUNC void solveInPlace(const MatrixBase<OtherDerived>& other) const { return solveInPlace<OnTheLeft>(other); } - EIGEN_DEVICE_FUNC - const SelfAdjointView<MatrixTypeNestedNonRef,Mode> selfadjointView() const - { - EIGEN_STATIC_ASSERT((Mode&UnitDiag)==0,PROGRAMMING_ERROR); - return SelfAdjointView<MatrixTypeNestedNonRef,Mode>(m_matrix); - } - EIGEN_DEVICE_FUNC - SelfAdjointView<MatrixTypeNestedNonRef,Mode> selfadjointView() - { - EIGEN_STATIC_ASSERT((Mode&UnitDiag)==0,PROGRAMMING_ERROR); - return SelfAdjointView<MatrixTypeNestedNonRef,Mode>(m_matrix); - } - template<typename OtherDerived> EIGEN_DEVICE_FUNC void swap(TriangularBase<OtherDerived> const & other) { - TriangularView<SwapWrapper<MatrixType>,Mode>(const_cast<MatrixType&>(m_matrix)).lazyAssign(other.derived()); + #ifdef EIGEN_TEST_EVALUATORS + call_assignment(derived(), other.const_cast_derived(), internal::swap_assign_op<Scalar>()); + #else + TriangularView<SwapWrapper<MatrixType>,Mode>(const_cast<MatrixType&>(derived().nestedExpression())).lazyAssign(other.const_cast_derived().nestedExpression()); + #endif } + // TODO: this overload is ambiguous and it should be deprecated (Gael) template<typename OtherDerived> EIGEN_DEVICE_FUNC void swap(MatrixBase<OtherDerived> const & other) { - SwapWrapper<MatrixType> swaper(const_cast<MatrixType&>(m_matrix)); + #ifdef EIGEN_TEST_EVALUATORS + call_assignment(derived(), other.const_cast_derived(), internal::swap_assign_op<Scalar>()); + #else + SwapWrapper<MatrixType> swaper(const_cast<MatrixType&>(derived().nestedExpression())); TriangularView<SwapWrapper<MatrixType>,Mode>(swaper).lazyAssign(other.derived()); + #endif } - - EIGEN_DEVICE_FUNC - Scalar determinant() const - { - if (Mode & UnitDiag) - return 1; - else if (Mode & ZeroDiag) - return 0; - else - return m_matrix.diagonal().prod(); - } + +#ifndef EIGEN_TEST_EVALUATORS // TODO simplify the following: template<typename ProductDerived, typename Lhs, typename Rhs> EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE TriangularView& operator=(const ProductBase<ProductDerived, Lhs,Rhs>& other) + EIGEN_STRONG_INLINE TriangularViewType& operator=(const ProductBase<ProductDerived, Lhs,Rhs>& other) { setZero(); return assignProduct(other,1); @@ -412,14 +531,14 @@ template<typename _MatrixType, unsigned int _Mode> class TriangularView template<typename ProductDerived, typename Lhs, typename Rhs> EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE TriangularView& operator+=(const ProductBase<ProductDerived, Lhs,Rhs>& other) + EIGEN_STRONG_INLINE TriangularViewType& operator+=(const ProductBase<ProductDerived, Lhs,Rhs>& other) { return assignProduct(other,1); } template<typename ProductDerived, typename Lhs, typename Rhs> EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE TriangularView& operator-=(const ProductBase<ProductDerived, Lhs,Rhs>& other) + EIGEN_STRONG_INLINE TriangularViewType& operator-=(const ProductBase<ProductDerived, Lhs,Rhs>& other) { return assignProduct(other,-1); } @@ -427,7 +546,7 @@ template<typename _MatrixType, unsigned int _Mode> class TriangularView template<typename ProductDerived> EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE TriangularView& operator=(const ScaledProduct<ProductDerived>& other) + EIGEN_STRONG_INLINE TriangularViewType& operator=(const ScaledProduct<ProductDerived>& other) { setZero(); return assignProduct(other,other.alpha()); @@ -435,25 +554,41 @@ template<typename _MatrixType, unsigned int _Mode> class TriangularView template<typename ProductDerived> EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE TriangularView& operator+=(const ScaledProduct<ProductDerived>& other) + EIGEN_STRONG_INLINE TriangularViewType& operator+=(const ScaledProduct<ProductDerived>& other) { return assignProduct(other,other.alpha()); } template<typename ProductDerived> EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE TriangularView& operator-=(const ScaledProduct<ProductDerived>& other) + EIGEN_STRONG_INLINE TriangularViewType& operator-=(const ScaledProduct<ProductDerived>& other) { return assignProduct(other,-other.alpha()); } + +#endif // EIGEN_TEST_EVALUATORS + +#ifdef EIGEN_TEST_EVALUATORS + + template<typename RhsType, typename DstType> + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE void _solve_impl(const RhsType &rhs, DstType &dst) const { + if(!(internal::is_same<RhsType,DstType>::value && internal::extract_data(dst) == internal::extract_data(rhs))) + dst = rhs; + this->template solveInPlace(dst); + } + + template<typename ProductType> + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE TriangularViewType& _assignProduct(const ProductType& prod, const Scalar& alpha); protected: - +#else + protected: template<typename ProductDerived, typename Lhs, typename Rhs> EIGEN_DEVICE_FUNC - EIGEN_STRONG_INLINE TriangularView& assignProduct(const ProductBase<ProductDerived, Lhs,Rhs>& prod, const Scalar& alpha); - - MatrixTypeNested m_matrix; + EIGEN_STRONG_INLINE TriangularViewType& assignProduct(const ProductBase<ProductDerived, Lhs,Rhs>& prod, const Scalar& alpha); +#endif }; /*************************************************************************** @@ -462,6 +597,8 @@ template<typename _MatrixType, unsigned int _Mode> class TriangularView namespace internal { +#ifndef EIGEN_TEST_EVALUATORS + template<typename Derived1, typename Derived2, unsigned int Mode, int UnrollCount, bool ClearOpposite> struct triangular_assignment_selector { @@ -626,13 +763,57 @@ struct triangular_assignment_selector<Derived1, Derived2, UnitLower, Dynamic, Cl } }; +#endif // EIGEN_TEST_EVALUATORS + } // end namespace internal +#ifdef EIGEN_TEST_EVALUATORS + // FIXME should we keep that possibility template<typename MatrixType, unsigned int Mode> template<typename OtherDerived> inline TriangularView<MatrixType, Mode>& -TriangularView<MatrixType, Mode>::operator=(const MatrixBase<OtherDerived>& other) +TriangularViewImpl<MatrixType, Mode, Dense>::operator=(const MatrixBase<OtherDerived>& other) +{ + internal::call_assignment_no_alias(derived(), other.derived(), internal::assign_op<Scalar>()); + return derived(); +} + +// FIXME should we keep that possibility +template<typename MatrixType, unsigned int Mode> +template<typename OtherDerived> +void TriangularViewImpl<MatrixType, Mode, Dense>::lazyAssign(const MatrixBase<OtherDerived>& other) +{ + internal::call_assignment(derived().noalias(), other.template triangularView<Mode>()); +} + + + +template<typename MatrixType, unsigned int Mode> +template<typename OtherDerived> +inline TriangularView<MatrixType, Mode>& +TriangularViewImpl<MatrixType, Mode, Dense>::operator=(const TriangularBase<OtherDerived>& other) +{ + eigen_assert(Mode == int(OtherDerived::Mode)); + internal::call_assignment(derived(), other.derived()); + return derived(); +} + +template<typename MatrixType, unsigned int Mode> +template<typename OtherDerived> +void TriangularViewImpl<MatrixType, Mode, Dense>::lazyAssign(const TriangularBase<OtherDerived>& other) +{ + eigen_assert(Mode == int(OtherDerived::Mode)); + internal::call_assignment(derived().noalias(), other.derived()); +} + +#else + +// FIXME should we keep that possibility +template<typename MatrixType, unsigned int Mode> +template<typename OtherDerived> +inline TriangularView<MatrixType, Mode>& +TriangularViewImpl<MatrixType, Mode, Dense>::operator=(const MatrixBase<OtherDerived>& other) { if(OtherDerived::Flags & EvalBeforeAssigningBit) { @@ -642,26 +823,26 @@ TriangularView<MatrixType, Mode>::operator=(const MatrixBase<OtherDerived>& othe } else lazyAssign(other.derived()); - return *this; + return derived(); } // FIXME should we keep that possibility template<typename MatrixType, unsigned int Mode> template<typename OtherDerived> -void TriangularView<MatrixType, Mode>::lazyAssign(const MatrixBase<OtherDerived>& other) +void TriangularViewImpl<MatrixType, Mode, Dense>::lazyAssign(const MatrixBase<OtherDerived>& other) { enum { unroll = MatrixType::SizeAtCompileTime != Dynamic && internal::traits<OtherDerived>::CoeffReadCost != Dynamic && MatrixType::SizeAtCompileTime*internal::traits<OtherDerived>::CoeffReadCost/2 <= EIGEN_UNROLLING_LIMIT }; - eigen_assert(m_matrix.rows() == other.rows() && m_matrix.cols() == other.cols()); + eigen_assert(derived().rows() == other.rows() && derived().cols() == other.cols()); internal::triangular_assignment_selector <MatrixType, OtherDerived, int(Mode), unroll ? int(MatrixType::SizeAtCompileTime) : Dynamic, false // do not change the opposite triangular part - >::run(m_matrix.const_cast_derived(), other.derived()); + >::run(derived().nestedExpression().const_cast_derived(), other.derived()); } @@ -669,7 +850,7 @@ void TriangularView<MatrixType, Mode>::lazyAssign(const MatrixBase<OtherDerived> template<typename MatrixType, unsigned int Mode> template<typename OtherDerived> inline TriangularView<MatrixType, Mode>& -TriangularView<MatrixType, Mode>::operator=(const TriangularBase<OtherDerived>& other) +TriangularViewImpl<MatrixType, Mode, Dense>::operator=(const TriangularBase<OtherDerived>& other) { eigen_assert(Mode == int(OtherDerived::Mode)); if(internal::traits<OtherDerived>::Flags & EvalBeforeAssigningBit) @@ -680,12 +861,12 @@ TriangularView<MatrixType, Mode>::operator=(const TriangularBase<OtherDerived>& } else lazyAssign(other.derived().nestedExpression()); - return *this; + return derived(); } template<typename MatrixType, unsigned int Mode> template<typename OtherDerived> -void TriangularView<MatrixType, Mode>::lazyAssign(const TriangularBase<OtherDerived>& other) +void TriangularViewImpl<MatrixType, Mode, Dense>::lazyAssign(const TriangularBase<OtherDerived>& other) { enum { unroll = MatrixType::SizeAtCompileTime != Dynamic @@ -693,15 +874,17 @@ void TriangularView<MatrixType, Mode>::lazyAssign(const TriangularBase<OtherDeri && MatrixType::SizeAtCompileTime * internal::traits<OtherDerived>::CoeffReadCost / 2 <= EIGEN_UNROLLING_LIMIT }; - eigen_assert(m_matrix.rows() == other.rows() && m_matrix.cols() == other.cols()); + eigen_assert(derived().rows() == other.rows() && derived().cols() == other.cols()); internal::triangular_assignment_selector <MatrixType, OtherDerived, int(Mode), unroll ? int(MatrixType::SizeAtCompileTime) : Dynamic, false // preserve the opposite triangular part - >::run(m_matrix.const_cast_derived(), other.derived().nestedExpression()); + >::run(derived().nestedExpression().const_cast_derived(), other.derived().nestedExpression()); } +#endif // EIGEN_TEST_EVALUATORS + /*************************************************************************** * Implementation of TriangularBase methods ***************************************************************************/ @@ -722,6 +905,8 @@ void TriangularBase<Derived>::evalTo(MatrixBase<DenseDerived> &other) const evalToLazy(other.derived()); } +#ifndef EIGEN_TEST_EVALUATORS + /** Assigns a triangular or selfadjoint matrix to a dense matrix. * If the matrix is triangular, the opposite part is set to zero. */ template<typename Derived> @@ -743,6 +928,8 @@ void TriangularBase<Derived>::evalToLazy(MatrixBase<DenseDerived> &other) const >::run(other.derived(), derived().nestedExpression()); } +#endif // EIGEN_TEST_EVALUATORS + /*************************************************************************** * Implementation of TriangularView methods ***************************************************************************/ @@ -831,6 +1018,300 @@ bool MatrixBase<Derived>::isLowerTriangular(const RealScalar& prec) const return true; } + +#ifdef EIGEN_ENABLE_EVALUATORS + +/*************************************************************************** +**************************************************************************** +* Evaluators and Assignment of triangular expressions +*************************************************************************** +***************************************************************************/ + +namespace internal { + + +// TODO currently a triangular expression has the form TriangularView<.,.> +// in the future triangular-ness should be defined by the expression traits +// such that Transpose<TriangularView<.,.> > is valid. (currently TriangularBase::transpose() is overloaded to make it work) +template<typename MatrixType, unsigned int Mode> +struct evaluator_traits<TriangularView<MatrixType,Mode> > +{ + typedef typename storage_kind_to_evaluator_kind<typename MatrixType::StorageKind>::Kind Kind; + typedef typename glue_shapes<typename evaluator_traits<MatrixType>::Shape, TriangularShape>::type Shape; + + // 1 if assignment A = B assumes aliasing when B is of type T and thus B needs to be evaluated into a + // temporary; 0 if not. + static const int AssumeAliasing = 0; +}; + +template<typename MatrixType, unsigned int Mode> +struct unary_evaluator<TriangularView<MatrixType,Mode>, IndexBased> + : evaluator<typename internal::remove_all<MatrixType>::type> +{ + typedef TriangularView<MatrixType,Mode> XprType; + typedef evaluator<typename internal::remove_all<MatrixType>::type> Base; + typedef evaluator<XprType> type; + unary_evaluator(const XprType &xpr) : Base(xpr.nestedExpression()) {} +}; + +// Additional assignment kinds: +struct Triangular2Triangular {}; +struct Triangular2Dense {}; +struct Dense2Triangular {}; + + +template<typename Kernel, unsigned int Mode, int UnrollCount, bool ClearOpposite> struct triangular_assignment_loop; + + +/** \internal Specialization of the dense assignment kernel for triangular matrices. + * The main difference is that the triangular, diagonal, and opposite parts are processed through three different functions. + * \tparam UpLo must be either Lower or Upper + * \tparam Mode must be either 0, UnitDiag, ZeroDiag, or SelfAdjoint + */ +template<int UpLo, int Mode, int SetOpposite, typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT, typename Functor, int Version = Specialized> +class triangular_dense_assignment_kernel : public generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, Version> +{ +protected: + typedef generic_dense_assignment_kernel<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor, Version> Base; + typedef typename Base::DstXprType DstXprType; + typedef typename Base::SrcXprType SrcXprType; + using Base::m_dst; + using Base::m_src; + using Base::m_functor; +public: + + typedef typename Base::DstEvaluatorType DstEvaluatorType; + typedef typename Base::SrcEvaluatorType SrcEvaluatorType; + typedef typename Base::Scalar Scalar; + typedef typename Base::Index Index; + typedef typename Base::AssignmentTraits AssignmentTraits; + + + triangular_dense_assignment_kernel(DstEvaluatorType &dst, const SrcEvaluatorType &src, const Functor &func, DstXprType& dstExpr) + : Base(dst, src, func, dstExpr) + {} + +#ifdef EIGEN_INTERNAL_DEBUGGING + void assignCoeff(Index row, Index col) + { + eigen_internal_assert(row!=col); + Base::assignCoeff(row,col); + } +#else + using Base::assignCoeff; +#endif + + void assignDiagonalCoeff(Index id) + { + if(Mode==UnitDiag && SetOpposite) m_functor.assignCoeff(m_dst.coeffRef(id,id), Scalar(1)); + else if(Mode==ZeroDiag && SetOpposite) m_functor.assignCoeff(m_dst.coeffRef(id,id), Scalar(0)); + else if(Mode==0) Base::assignCoeff(id,id); + } + + void assignOppositeCoeff(Index row, Index col) + { + eigen_internal_assert(row!=col); + if(SetOpposite) + m_functor.assignCoeff(m_dst.coeffRef(row,col), Scalar(0)); + } +}; + +template<int Mode, bool SetOpposite, typename DstXprType, typename SrcXprType, typename Functor> +void call_triangular_assignment_loop(const DstXprType& dst, const SrcXprType& src, const Functor &func) +{ + eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); + + typedef typename evaluator<DstXprType>::type DstEvaluatorType; + typedef typename evaluator<SrcXprType>::type SrcEvaluatorType; + + DstEvaluatorType dstEvaluator(dst); + SrcEvaluatorType srcEvaluator(src); + + typedef triangular_dense_assignment_kernel< Mode&(Lower|Upper),Mode&(UnitDiag|ZeroDiag|SelfAdjoint),SetOpposite, + DstEvaluatorType,SrcEvaluatorType,Functor> Kernel; + Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived()); + + enum { + unroll = DstXprType::SizeAtCompileTime != Dynamic + && SrcEvaluatorType::CoeffReadCost != Dynamic + && DstXprType::SizeAtCompileTime * SrcEvaluatorType::CoeffReadCost / 2 <= EIGEN_UNROLLING_LIMIT + }; + + triangular_assignment_loop<Kernel, Mode, unroll ? int(DstXprType::SizeAtCompileTime) : Dynamic, SetOpposite>::run(kernel); +} + +template<int Mode, bool SetOpposite, typename DstXprType, typename SrcXprType> +void call_triangular_assignment_loop(const DstXprType& dst, const SrcXprType& src) +{ + call_triangular_assignment_loop<Mode,SetOpposite>(dst, src, internal::assign_op<typename DstXprType::Scalar>()); +} + +template<> struct AssignmentKind<TriangularShape,TriangularShape> { typedef Triangular2Triangular Kind; }; +template<> struct AssignmentKind<DenseShape,TriangularShape> { typedef Triangular2Dense Kind; }; +template<> struct AssignmentKind<TriangularShape,DenseShape> { typedef Dense2Triangular Kind; }; + + +template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> +struct Assignment<DstXprType, SrcXprType, Functor, Triangular2Triangular, Scalar> +{ + static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) + { + eigen_assert(int(DstXprType::Mode) == int(SrcXprType::Mode)); + + call_triangular_assignment_loop<DstXprType::Mode, false>(dst, src, func); + } +}; + +template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> +struct Assignment<DstXprType, SrcXprType, Functor, Triangular2Dense, Scalar> +{ + static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) + { + call_triangular_assignment_loop<SrcXprType::Mode, (SrcXprType::Mode&SelfAdjoint)==0>(dst, src, func); + } +}; + +template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> +struct Assignment<DstXprType, SrcXprType, Functor, Dense2Triangular, Scalar> +{ + static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) + { + call_triangular_assignment_loop<DstXprType::Mode, false>(dst, src, func); + } +}; + + +template<typename Kernel, unsigned int Mode, int UnrollCount, bool SetOpposite> +struct triangular_assignment_loop +{ + // FIXME: this is not very clean, perhaps this information should be provided by the kernel? + typedef typename Kernel::DstEvaluatorType DstEvaluatorType; + typedef typename DstEvaluatorType::XprType DstXprType; + + enum { + col = (UnrollCount-1) / DstXprType::RowsAtCompileTime, + row = (UnrollCount-1) % DstXprType::RowsAtCompileTime + }; + + typedef typename Kernel::Scalar Scalar; + + EIGEN_DEVICE_FUNC + static inline void run(Kernel &kernel) + { + triangular_assignment_loop<Kernel, Mode, UnrollCount-1, SetOpposite>::run(kernel); + + if(row==col) + kernel.assignDiagonalCoeff(row); + else if( ((Mode&Lower) && row>col) || ((Mode&Upper) && row<col) ) + kernel.assignCoeff(row,col); + else if(SetOpposite) + kernel.assignOppositeCoeff(row,col); + } +}; + +// prevent buggy user code from causing an infinite recursion +template<typename Kernel, unsigned int Mode, bool SetOpposite> +struct triangular_assignment_loop<Kernel, Mode, 0, SetOpposite> +{ + EIGEN_DEVICE_FUNC + static inline void run(Kernel &) {} +}; + + + +// TODO: experiment with a recursive assignment procedure splitting the current +// triangular part into one rectangular and two triangular parts. + + +template<typename Kernel, unsigned int Mode, bool SetOpposite> +struct triangular_assignment_loop<Kernel, Mode, Dynamic, SetOpposite> +{ + typedef typename Kernel::Index Index; + typedef typename Kernel::Scalar Scalar; + EIGEN_DEVICE_FUNC + static inline void run(Kernel &kernel) + { + for(Index j = 0; j < kernel.cols(); ++j) + { + Index maxi = (std::min)(j, kernel.rows()); + Index i = 0; + if (((Mode&Lower) && SetOpposite) || (Mode&Upper)) + { + for(; i < maxi; ++i) + if(Mode&Upper) kernel.assignCoeff(i, j); + else kernel.assignOppositeCoeff(i, j); + } + else + i = maxi; + + if(i<kernel.rows()) // then i==j + kernel.assignDiagonalCoeff(i++); + + if (((Mode&Upper) && SetOpposite) || (Mode&Lower)) + { + for(; i < kernel.rows(); ++i) + if(Mode&Lower) kernel.assignCoeff(i, j); + else kernel.assignOppositeCoeff(i, j); + } + } + } +}; + +} // end namespace internal + +#ifdef EIGEN_TEST_EVALUATORS +/** Assigns a triangular or selfadjoint matrix to a dense matrix. + * If the matrix is triangular, the opposite part is set to zero. */ +template<typename Derived> +template<typename DenseDerived> +void TriangularBase<Derived>::evalToLazy(MatrixBase<DenseDerived> &other) const +{ + other.derived().resize(this->rows(), this->cols()); + internal::call_triangular_assignment_loop<Derived::Mode,(Derived::Mode&SelfAdjoint)==0 /* SetOpposite */>(other.derived(), derived().nestedExpression()); +} + +namespace internal { + +// Triangular = Product +template< typename DstXprType, typename Lhs, typename Rhs, typename Scalar> +struct Assignment<DstXprType, Product<Lhs,Rhs,DefaultProduct>, internal::assign_op<Scalar>, Dense2Triangular, Scalar> +{ + typedef Product<Lhs,Rhs,DefaultProduct> SrcXprType; + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + { + dst.setZero(); + dst._assignProduct(src, 1); + } +}; + +// Triangular += Product +template< typename DstXprType, typename Lhs, typename Rhs, typename Scalar> +struct Assignment<DstXprType, Product<Lhs,Rhs,DefaultProduct>, internal::add_assign_op<Scalar>, Dense2Triangular, Scalar> +{ + typedef Product<Lhs,Rhs,DefaultProduct> SrcXprType; + static void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op<Scalar> &) + { + dst._assignProduct(src, 1); + } +}; + +// Triangular -= Product +template< typename DstXprType, typename Lhs, typename Rhs, typename Scalar> +struct Assignment<DstXprType, Product<Lhs,Rhs,DefaultProduct>, internal::sub_assign_op<Scalar>, Dense2Triangular, Scalar> +{ + typedef Product<Lhs,Rhs,DefaultProduct> SrcXprType; + static void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op<Scalar> &) + { + dst._assignProduct(src, -1); + } +}; + + +} // end namespace internal +#endif + +#endif // EIGEN_ENABLE_EVALUATORS + } // end namespace Eigen #endif // EIGEN_TRIANGULARMATRIX_H diff --git a/Eigen/src/Core/VectorwiseOp.h b/Eigen/src/Core/VectorwiseOp.h index 52eb4f604..1a9eead43 100644 --- a/Eigen/src/Core/VectorwiseOp.h +++ b/Eigen/src/Core/VectorwiseOp.h @@ -48,10 +48,15 @@ struct traits<PartialReduxExpr<MatrixType, MemberOp, Direction> > ColsAtCompileTime = Direction==Horizontal ? 1 : MatrixType::ColsAtCompileTime, MaxRowsAtCompileTime = Direction==Vertical ? 1 : MatrixType::MaxRowsAtCompileTime, MaxColsAtCompileTime = Direction==Horizontal ? 1 : MatrixType::MaxColsAtCompileTime, +#ifndef EIGEN_TEST_EVALUATORS Flags0 = (unsigned int)_MatrixTypeNested::Flags & HereditaryBits, Flags = (Flags0 & ~RowMajorBit) | (RowsAtCompileTime == 1 ? RowMajorBit : 0), +#else + Flags = RowsAtCompileTime == 1 ? RowMajorBit : 0, +#endif TraversalSize = Direction==Vertical ? MatrixType::RowsAtCompileTime : MatrixType::ColsAtCompileTime }; +#ifndef EIGEN_TEST_EVALUATORS #if EIGEN_GNUC_AT_LEAST(3,4) typedef typename MemberOp::template Cost<InputScalar,int(TraversalSize)> CostOpType; #else @@ -61,6 +66,7 @@ struct traits<PartialReduxExpr<MatrixType, MemberOp, Direction> > CoeffReadCost = TraversalSize==Dynamic ? Dynamic : TraversalSize * traits<_MatrixTypeNested>::CoeffReadCost + int(CostOpType::value) }; +#endif }; } diff --git a/Eigen/src/Core/Visitor.h b/Eigen/src/Core/Visitor.h index 6f4b9ec35..76d452d9a 100644 --- a/Eigen/src/Core/Visitor.h +++ b/Eigen/src/Core/Visitor.h @@ -53,6 +53,35 @@ struct visitor_impl<Visitor, Derived, Dynamic> } }; +#ifdef EIGEN_ENABLE_EVALUATORS +// evaluator adaptor +template<typename XprType> +class visitor_evaluator +{ +public: + visitor_evaluator(const XprType &xpr) : m_evaluator(xpr), m_xpr(xpr) {} + + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + typedef typename XprType::CoeffReturnType CoeffReturnType; + + enum { + RowsAtCompileTime = XprType::RowsAtCompileTime, + CoeffReadCost = internal::evaluator<XprType>::CoeffReadCost + }; + + Index rows() const { return m_xpr.rows(); } + Index cols() const { return m_xpr.cols(); } + Index size() const { return m_xpr.size(); } + + CoeffReturnType coeff(Index row, Index col) const + { return m_evaluator.coeff(row, col); } + +protected: + typename internal::evaluator<XprType>::nestedType m_evaluator; + const XprType &m_xpr; +}; +#endif } // end namespace internal /** Applies the visitor \a visitor to the whole coefficients of the matrix or vector. @@ -76,6 +105,19 @@ template<typename Derived> template<typename Visitor> void DenseBase<Derived>::visit(Visitor& visitor) const { +#ifdef EIGEN_TEST_EVALUATORS + typedef typename internal::visitor_evaluator<Derived> ThisEvaluator; + ThisEvaluator thisEval(derived()); + + enum { unroll = SizeAtCompileTime != Dynamic + && ThisEvaluator::CoeffReadCost != Dynamic + && (SizeAtCompileTime == 1 || internal::functor_traits<Visitor>::Cost != Dynamic) + && SizeAtCompileTime * ThisEvaluator::CoeffReadCost + (SizeAtCompileTime-1) * internal::functor_traits<Visitor>::Cost + <= EIGEN_UNROLLING_LIMIT }; + return internal::visitor_impl<Visitor, ThisEvaluator, + unroll ? int(SizeAtCompileTime) : Dynamic + >::run(thisEval, visitor); +#else enum { unroll = SizeAtCompileTime != Dynamic && CoeffReadCost != Dynamic && (SizeAtCompileTime == 1 || internal::functor_traits<Visitor>::Cost != Dynamic) @@ -84,6 +126,7 @@ void DenseBase<Derived>::visit(Visitor& visitor) const return internal::visitor_impl<Visitor, Derived, unroll ? int(SizeAtCompileTime) : Dynamic >::run(derived(), visitor); +#endif } namespace internal { diff --git a/Eigen/src/Core/functors/AssignmentFunctors.h b/Eigen/src/Core/functors/AssignmentFunctors.h index ae264aa64..d4d85a1ca 100644 --- a/Eigen/src/Core/functors/AssignmentFunctors.h +++ b/Eigen/src/Core/functors/AssignmentFunctors.h @@ -31,7 +31,7 @@ template<typename Scalar> struct functor_traits<assign_op<Scalar> > { enum { Cost = NumTraits<Scalar>::ReadCost, - PacketAccess = packet_traits<Scalar>::IsVectorized + PacketAccess = packet_traits<Scalar>::Vectorizable }; }; @@ -73,7 +73,7 @@ template<typename Scalar> struct functor_traits<sub_assign_op<Scalar> > { enum { Cost = NumTraits<Scalar>::ReadCost + NumTraits<Scalar>::AddCost, - PacketAccess = packet_traits<Scalar>::HasAdd + PacketAccess = packet_traits<Scalar>::HasSub }; }; @@ -81,22 +81,24 @@ struct functor_traits<sub_assign_op<Scalar> > { * \brief Template functor for scalar/packet assignment with multiplication * */ -template<typename Scalar> struct mul_assign_op { +template<typename DstScalar, typename SrcScalar=DstScalar> +struct mul_assign_op { EIGEN_EMPTY_STRUCT_CTOR(mul_assign_op) - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Scalar& a, const Scalar& b) const { a *= b; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(DstScalar& a, const SrcScalar& b) const { a *= b; } template<int Alignment, typename Packet> - EIGEN_STRONG_INLINE void assignPacket(Scalar* a, const Packet& b) const - { internal::pstoret<Scalar,Packet,Alignment>(a,internal::pmul(internal::ploadt<Packet,Alignment>(a),b)); } + EIGEN_STRONG_INLINE void assignPacket(DstScalar* a, const Packet& b) const + { internal::pstoret<DstScalar,Packet,Alignment>(a,internal::pmul(internal::ploadt<Packet,Alignment>(a),b)); } }; -template<typename Scalar> -struct functor_traits<mul_assign_op<Scalar> > { +template<typename DstScalar, typename SrcScalar> +struct functor_traits<mul_assign_op<DstScalar,SrcScalar> > { enum { - Cost = NumTraits<Scalar>::ReadCost + NumTraits<Scalar>::MulCost, - PacketAccess = packet_traits<Scalar>::HasMul + Cost = NumTraits<DstScalar>::ReadCost + NumTraits<DstScalar>::MulCost, + PacketAccess = is_same<DstScalar,SrcScalar>::value && packet_traits<DstScalar>::HasMul }; }; +template<typename DstScalar,typename SrcScalar> struct functor_is_product_like<mul_assign_op<DstScalar,SrcScalar> > { enum { ret = 1 }; }; /** \internal * \brief Template functor for scalar/packet assignment with diviving @@ -115,7 +117,7 @@ template<typename Scalar> struct functor_traits<div_assign_op<Scalar> > { enum { Cost = NumTraits<Scalar>::ReadCost + NumTraits<Scalar>::MulCost, - PacketAccess = packet_traits<Scalar>::HasMul + PacketAccess = packet_traits<Scalar>::HasDiv }; }; @@ -156,7 +158,7 @@ template<typename Scalar> struct functor_traits<swap_assign_op<Scalar> > { enum { Cost = 3 * NumTraits<Scalar>::ReadCost, - PacketAccess = packet_traits<Scalar>::IsVectorized + PacketAccess = packet_traits<Scalar>::Vectorizable }; }; diff --git a/Eigen/src/Core/products/CoeffBasedProduct.h b/Eigen/src/Core/products/CoeffBasedProduct.h index 637513132..76806fd62 100644 --- a/Eigen/src/Core/products/CoeffBasedProduct.h +++ b/Eigen/src/Core/products/CoeffBasedProduct.h @@ -13,8 +13,9 @@ namespace Eigen { +#ifndef EIGEN_TEST_EVALUATORS namespace internal { - + /********************************************************************************* * Coefficient based product implementation. * It is designed for the following use cases: @@ -41,16 +42,15 @@ struct traits<CoeffBasedProduct<LhsNested,RhsNested,NestingFlags> > typedef typename remove_all<LhsNested>::type _LhsNested; typedef typename remove_all<RhsNested>::type _RhsNested; typedef typename scalar_product_traits<typename _LhsNested::Scalar, typename _RhsNested::Scalar>::ReturnType Scalar; - typedef typename promote_storage_type<typename traits<_LhsNested>::StorageKind, - typename traits<_RhsNested>::StorageKind>::ret StorageKind; + typedef typename product_promote_storage_type<typename traits<_LhsNested>::StorageKind, + typename traits<_RhsNested>::StorageKind, + 0>::ret StorageKind; typedef typename promote_index_type<typename traits<_LhsNested>::Index, typename traits<_RhsNested>::Index>::type Index; enum { - LhsCoeffReadCost = _LhsNested::CoeffReadCost, - RhsCoeffReadCost = _RhsNested::CoeffReadCost, - LhsFlags = _LhsNested::Flags, - RhsFlags = _RhsNested::Flags, + LhsFlags = traits<_LhsNested>::Flags, + RhsFlags = traits<_RhsNested>::Flags, RowsAtCompileTime = _LhsNested::RowsAtCompileTime, ColsAtCompileTime = _RhsNested::ColsAtCompileTime, @@ -89,11 +89,13 @@ struct traits<CoeffBasedProduct<LhsNested,RhsNested,NestingFlags> > | (CanVectorizeRhs ? (RhsFlags & AlignedBit) : 0) // TODO enable vectorization for mixed types | (SameType && (CanVectorizeLhs || CanVectorizeRhs) ? PacketAccessBit : 0), - - CoeffReadCost = InnerSize == Dynamic ? Dynamic +#ifndef EIGEN_TEST_EVALUATORS + LhsCoeffReadCost = traits<_LhsNested>::CoeffReadCost, + RhsCoeffReadCost = traits<_RhsNested>::CoeffReadCost, + CoeffReadCost = (InnerSize == Dynamic || LhsCoeffReadCost==Dynamic || RhsCoeffReadCost==Dynamic || NumTraits<Scalar>::AddCost==Dynamic || NumTraits<Scalar>::MulCost==Dynamic) ? Dynamic : InnerSize * (NumTraits<Scalar>::MulCost + LhsCoeffReadCost + RhsCoeffReadCost) + (InnerSize - 1) * NumTraits<Scalar>::AddCost, - +#endif /* CanVectorizeInner deserves special explanation. It does not affect the product flags. It is not used outside * of Product. If the Product itself is not a packet-access expression, there is still a chance that the inner * loop of the product might be vectorized. This is the meaning of CanVectorizeInner. Since it doesn't affect @@ -110,6 +112,8 @@ struct traits<CoeffBasedProduct<LhsNested,RhsNested,NestingFlags> > } // end namespace internal +#ifndef EIGEN_TEST_EVALUATORS + template<typename LhsNested, typename RhsNested, int NestingFlags> class CoeffBasedProduct : internal::no_assignment_operator, @@ -447,6 +451,10 @@ struct product_packet_impl<ColMajor, Dynamic, Lhs, Rhs, Packet, LoadMode> } // end namespace internal +#endif // EIGEN_TEST_EVALUATORS + +#endif // <EIGEN_TEST_EVALUATORS + } // end namespace Eigen #endif // EIGEN_COEFFBASED_PRODUCT_H diff --git a/Eigen/src/Core/products/GeneralMatrixMatrix.h b/Eigen/src/Core/products/GeneralMatrixMatrix.h index 6ad07eccb..66a4fe536 100644 --- a/Eigen/src/Core/products/GeneralMatrixMatrix.h +++ b/Eigen/src/Core/products/GeneralMatrixMatrix.h @@ -216,8 +216,8 @@ struct gemm_functor cols = m_rhs.cols(); Gemm::run(rows, cols, m_lhs.cols(), - /*(const Scalar*)*/&m_lhs.coeffRef(row,0), m_lhs.outerStride(), - /*(const Scalar*)*/&m_rhs.coeffRef(0,col), m_rhs.outerStride(), + &m_lhs.coeffRef(row,0), m_lhs.outerStride(), + &m_rhs.coeffRef(0,col), m_rhs.outerStride(), (Scalar*)&(m_dest.coeffRef(row,col)), m_dest.outerStride(), m_actualAlpha, m_blocking, info); } @@ -367,6 +367,7 @@ class gemm_blocking_space<StorageOrder,_LhsScalar,_RhsScalar,MaxRows, MaxCols, M } // end namespace internal +#ifndef EIGEN_TEST_EVALUATORS template<typename Lhs, typename Rhs> class GeneralProduct<Lhs, Rhs, GemmProduct> : public ProductBase<GeneralProduct<Lhs,Rhs,GemmProduct>, Lhs, Rhs> @@ -444,6 +445,96 @@ class GeneralProduct<Lhs, Rhs, GemmProduct> internal::parallelize_gemm<(Dest::MaxRowsAtCompileTime>32 || Dest::MaxRowsAtCompileTime==Dynamic)>(GemmFunctor(lhs, rhs, dst, actualAlpha, blocking), this->rows(), this->cols(), Dest::Flags&RowMajorBit); } }; +#endif // EIGEN_TEST_EVALUATORS + +#ifdef EIGEN_ENABLE_EVALUATORS +namespace internal { + +template<typename Lhs, typename Rhs> +struct generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,GemmProduct> + : generic_product_impl_base<Lhs,Rhs,generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,GemmProduct> > +{ + typedef typename Product<Lhs,Rhs>::Scalar Scalar; + typedef typename Product<Lhs,Rhs>::Index Index; + typedef typename Lhs::Scalar LhsScalar; + typedef typename Rhs::Scalar RhsScalar; + + typedef internal::blas_traits<Lhs> LhsBlasTraits; + typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; + typedef typename internal::remove_all<ActualLhsType>::type ActualLhsTypeCleaned; + + typedef internal::blas_traits<Rhs> RhsBlasTraits; + typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; + typedef typename internal::remove_all<ActualRhsType>::type ActualRhsTypeCleaned; + + enum { + MaxDepthAtCompileTime = EIGEN_SIZE_MIN_PREFER_FIXED(Lhs::MaxColsAtCompileTime,Rhs::MaxRowsAtCompileTime) + }; + + typedef generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,CoeffBasedProductMode> lazyproduct; + + template<typename Dst> + static void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { + if((rhs.rows()+dst.rows()+dst.cols())<20 && rhs.rows()>0) + lazyproduct::evalTo(dst, lhs, rhs); + else + { + dst.setZero(); + scaleAndAddTo(dst, lhs, rhs, Scalar(1)); + } + } + + template<typename Dst> + static void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { + if((rhs.rows()+dst.rows()+dst.cols())<20 && rhs.rows()>0) + lazyproduct::addTo(dst, lhs, rhs); + else + scaleAndAddTo(dst,lhs, rhs, Scalar(1)); + } + + template<typename Dst> + static void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) + { + if((rhs.rows()+dst.rows()+dst.cols())<20 && rhs.rows()>0) + lazyproduct::subTo(dst, lhs, rhs); + else + scaleAndAddTo(dst, lhs, rhs, Scalar(-1)); + } + + template<typename Dest> + static void scaleAndAddTo(Dest& dst, const Lhs& a_lhs, const Rhs& a_rhs, const Scalar& alpha) + { + eigen_assert(dst.rows()==a_lhs.rows() && dst.cols()==a_rhs.cols()); + + typename internal::add_const_on_value_type<ActualLhsType>::type lhs = LhsBlasTraits::extract(a_lhs); + typename internal::add_const_on_value_type<ActualRhsType>::type rhs = RhsBlasTraits::extract(a_rhs); + + Scalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(a_lhs) + * RhsBlasTraits::extractScalarFactor(a_rhs); + + typedef internal::gemm_blocking_space<(Dest::Flags&RowMajorBit) ? RowMajor : ColMajor,LhsScalar,RhsScalar, + Dest::MaxRowsAtCompileTime,Dest::MaxColsAtCompileTime,MaxDepthAtCompileTime> BlockingType; + + typedef internal::gemm_functor< + Scalar, Index, + internal::general_matrix_matrix_product< + Index, + LhsScalar, (ActualLhsTypeCleaned::Flags&RowMajorBit) ? RowMajor : ColMajor, bool(LhsBlasTraits::NeedToConjugate), + RhsScalar, (ActualRhsTypeCleaned::Flags&RowMajorBit) ? RowMajor : ColMajor, bool(RhsBlasTraits::NeedToConjugate), + (Dest::Flags&RowMajorBit) ? RowMajor : ColMajor>, + ActualLhsTypeCleaned, ActualRhsTypeCleaned, Dest, BlockingType> GemmFunctor; + + BlockingType blocking(dst.rows(), dst.cols(), lhs.cols(), true); + + internal::parallelize_gemm<(Dest::MaxRowsAtCompileTime>32 || Dest::MaxRowsAtCompileTime==Dynamic)> + (GemmFunctor(lhs, rhs, dst, actualAlpha, blocking), a_lhs.rows(), a_rhs.cols(), Dest::Flags&RowMajorBit); + } +}; + +} // end namespace internal +#endif // EIGEN_ENABLE_EVALUATORS } // end namespace Eigen diff --git a/Eigen/src/Core/products/GeneralMatrixMatrixTriangular.h b/Eigen/src/Core/products/GeneralMatrixMatrixTriangular.h index 225b994d1..06c64714a 100644 --- a/Eigen/src/Core/products/GeneralMatrixMatrixTriangular.h +++ b/Eigen/src/Core/products/GeneralMatrixMatrixTriangular.h @@ -20,7 +20,7 @@ namespace internal { /********************************************************************** * This file implements a general A * B product while * evaluating only one triangular part of the product. -* This is more general version of self adjoint product (C += A A^T) +* This is a more general version of self adjoint product (C += A A^T) * as the level 3 SYRK Blas routine. **********************************************************************/ @@ -261,17 +261,30 @@ struct general_product_to_triangular_selector<MatrixType,ProductType,UpLo,false> } }; +#ifdef EIGEN_TEST_EVALUATORS +template<typename MatrixType, unsigned int UpLo> +template<typename ProductType> +TriangularView<MatrixType,UpLo>& TriangularViewImpl<MatrixType,UpLo,Dense>::_assignProduct(const ProductType& prod, const Scalar& alpha) +{ + eigen_assert(derived().nestedExpression().rows() == prod.rows() && derived().cols() == prod.cols()); + + general_product_to_triangular_selector<MatrixType, ProductType, UpLo, internal::traits<ProductType>::InnerSize==1>::run(derived().nestedExpression().const_cast_derived(), prod, alpha); + + return derived(); +} +#else template<typename MatrixType, unsigned int UpLo> template<typename ProductDerived, typename _Lhs, typename _Rhs> -TriangularView<MatrixType,UpLo>& TriangularView<MatrixType,UpLo>::assignProduct(const ProductBase<ProductDerived, _Lhs,_Rhs>& prod, const Scalar& alpha) +TriangularView<MatrixType,UpLo>& TriangularViewImpl<MatrixType,UpLo,Dense>::assignProduct(const ProductBase<ProductDerived, _Lhs,_Rhs>& prod, const Scalar& alpha) { - eigen_assert(m_matrix.rows() == prod.rows() && m_matrix.cols() == prod.cols()); + eigen_assert(derived().rows() == prod.rows() && derived().cols() == prod.cols()); - general_product_to_triangular_selector<MatrixType, ProductDerived, UpLo, (_Lhs::ColsAtCompileTime==1) || (_Rhs::RowsAtCompileTime==1)>::run(m_matrix.const_cast_derived(), prod.derived(), alpha); + general_product_to_triangular_selector<MatrixType, ProductDerived, UpLo, (_Lhs::ColsAtCompileTime==1) || (_Rhs::RowsAtCompileTime==1)> + ::run(derived().nestedExpression().const_cast_derived(), prod.derived(), alpha); - return *this; + return derived(); } - +#endif } // end namespace Eigen #endif // EIGEN_GENERAL_MATRIX_MATRIX_TRIANGULAR_H diff --git a/Eigen/src/Core/products/SelfadjointMatrixMatrix.h b/Eigen/src/Core/products/SelfadjointMatrixMatrix.h index d67164ec3..0ab3f3a56 100644 --- a/Eigen/src/Core/products/SelfadjointMatrixMatrix.h +++ b/Eigen/src/Core/products/SelfadjointMatrixMatrix.h @@ -459,6 +459,7 @@ EIGEN_DONT_INLINE void product_selfadjoint_matrix<Scalar,Index,LhsStorageOrder,f * Wrapper to product_selfadjoint_matrix ***************************************************************************/ +#ifndef EIGEN_TEST_EVALUATORS namespace internal { template<typename Lhs, int LhsMode, typename Rhs, int RhsMode> struct traits<SelfadjointProductMatrix<Lhs,LhsMode,false,Rhs,RhsMode,false> > @@ -508,6 +509,58 @@ struct SelfadjointProductMatrix<Lhs,LhsMode,false,Rhs,RhsMode,false> ); } }; +#endif // EIGEN_TEST_EVALUATORS +#ifdef EIGEN_ENABLE_EVALUATORS +namespace internal { + +template<typename Lhs, int LhsMode, typename Rhs, int RhsMode> +struct selfadjoint_product_impl<Lhs,LhsMode,false,Rhs,RhsMode,false> +{ + typedef typename Product<Lhs,Rhs>::Scalar Scalar; + typedef typename Product<Lhs,Rhs>::Index Index; + + typedef internal::blas_traits<Lhs> LhsBlasTraits; + typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; + typedef internal::blas_traits<Rhs> RhsBlasTraits; + typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; + + enum { + LhsIsUpper = (LhsMode&(Upper|Lower))==Upper, + LhsIsSelfAdjoint = (LhsMode&SelfAdjoint)==SelfAdjoint, + RhsIsUpper = (RhsMode&(Upper|Lower))==Upper, + RhsIsSelfAdjoint = (RhsMode&SelfAdjoint)==SelfAdjoint + }; + + template<typename Dest> + static void run(Dest &dst, const Lhs &a_lhs, const Rhs &a_rhs, const Scalar& alpha) + { + eigen_assert(dst.rows()==a_lhs.rows() && dst.cols()==a_rhs.cols()); + + typename internal::add_const_on_value_type<ActualLhsType>::type lhs = LhsBlasTraits::extract(a_lhs); + typename internal::add_const_on_value_type<ActualRhsType>::type rhs = RhsBlasTraits::extract(a_rhs); + + Scalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(a_lhs) + * RhsBlasTraits::extractScalarFactor(a_rhs); + + internal::product_selfadjoint_matrix<Scalar, Index, + EIGEN_LOGICAL_XOR(LhsIsUpper,internal::traits<Lhs>::Flags &RowMajorBit) ? RowMajor : ColMajor, LhsIsSelfAdjoint, + NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(LhsIsUpper,bool(LhsBlasTraits::NeedToConjugate)), + EIGEN_LOGICAL_XOR(RhsIsUpper,internal::traits<Rhs>::Flags &RowMajorBit) ? RowMajor : ColMajor, RhsIsSelfAdjoint, + NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(RhsIsUpper,bool(RhsBlasTraits::NeedToConjugate)), + internal::traits<Dest>::Flags&RowMajorBit ? RowMajor : ColMajor> + ::run( + lhs.rows(), rhs.cols(), // sizes + &lhs.coeffRef(0,0), lhs.outerStride(), // lhs info + &rhs.coeffRef(0,0), rhs.outerStride(), // rhs info + &dst.coeffRef(0,0), dst.outerStride(), // result info + actualAlpha // alpha + ); + } +}; + +} // end namespace internal + +#endif // EIGEN_ENABLE_EVALUATORS } // end namespace Eigen diff --git a/Eigen/src/Core/products/SelfadjointMatrixVector.h b/Eigen/src/Core/products/SelfadjointMatrixVector.h index 26e787949..020205c12 100644 --- a/Eigen/src/Core/products/SelfadjointMatrixVector.h +++ b/Eigen/src/Core/products/SelfadjointMatrixVector.h @@ -168,6 +168,7 @@ EIGEN_DONT_INLINE void selfadjoint_matrix_vector_product<Scalar,Index,StorageOrd * Wrapper to product_selfadjoint_vector ***************************************************************************/ +#ifndef EIGEN_TEST_EVALUATORS namespace internal { template<typename Lhs, int LhsMode, typename Rhs> struct traits<SelfadjointProductMatrix<Lhs,LhsMode,false,Rhs,0,true> > @@ -276,6 +277,109 @@ struct SelfadjointProductMatrix<Lhs,0,true,Rhs,RhsMode,false> } }; +#else // EIGEN_TEST_EVALUATORS + +namespace internal { + +template<typename Lhs, int LhsMode, typename Rhs> +struct selfadjoint_product_impl<Lhs,LhsMode,false,Rhs,0,true> +{ + typedef typename Product<Lhs,Rhs>::Scalar Scalar; + typedef typename Product<Lhs,Rhs>::Index Index; + + typedef internal::blas_traits<Lhs> LhsBlasTraits; + typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; + typedef typename internal::remove_all<ActualLhsType>::type ActualLhsTypeCleaned; + + typedef internal::blas_traits<Rhs> RhsBlasTraits; + typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; + typedef typename internal::remove_all<ActualRhsType>::type ActualRhsTypeCleaned; + + enum { LhsUpLo = LhsMode&(Upper|Lower) }; + + template<typename Dest> + static void run(Dest& dest, const Lhs &a_lhs, const Rhs &a_rhs, const Scalar& alpha) + { + typedef typename Dest::Scalar ResScalar; + typedef typename Rhs::Scalar RhsScalar; + typedef Map<Matrix<ResScalar,Dynamic,1>, Aligned> MappedDest; + + eigen_assert(dest.rows()==a_lhs.rows() && dest.cols()==a_rhs.cols()); + + typename internal::add_const_on_value_type<ActualLhsType>::type lhs = LhsBlasTraits::extract(a_lhs); + typename internal::add_const_on_value_type<ActualRhsType>::type rhs = RhsBlasTraits::extract(a_rhs); + + Scalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(a_lhs) + * RhsBlasTraits::extractScalarFactor(a_rhs); + + enum { + EvalToDest = (Dest::InnerStrideAtCompileTime==1), + UseRhs = (ActualRhsTypeCleaned::InnerStrideAtCompileTime==1) + }; + + internal::gemv_static_vector_if<ResScalar,Dest::SizeAtCompileTime,Dest::MaxSizeAtCompileTime,!EvalToDest> static_dest; + internal::gemv_static_vector_if<RhsScalar,ActualRhsTypeCleaned::SizeAtCompileTime,ActualRhsTypeCleaned::MaxSizeAtCompileTime,!UseRhs> static_rhs; + + ei_declare_aligned_stack_constructed_variable(ResScalar,actualDestPtr,dest.size(), + EvalToDest ? dest.data() : static_dest.data()); + + ei_declare_aligned_stack_constructed_variable(RhsScalar,actualRhsPtr,rhs.size(), + UseRhs ? const_cast<RhsScalar*>(rhs.data()) : static_rhs.data()); + + if(!EvalToDest) + { + #ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN + int size = dest.size(); + EIGEN_DENSE_STORAGE_CTOR_PLUGIN + #endif + MappedDest(actualDestPtr, dest.size()) = dest; + } + + if(!UseRhs) + { + #ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN + int size = rhs.size(); + EIGEN_DENSE_STORAGE_CTOR_PLUGIN + #endif + Map<typename ActualRhsTypeCleaned::PlainObject>(actualRhsPtr, rhs.size()) = rhs; + } + + + internal::selfadjoint_matrix_vector_product<Scalar, Index, (internal::traits<ActualLhsTypeCleaned>::Flags&RowMajorBit) ? RowMajor : ColMajor, + int(LhsUpLo), bool(LhsBlasTraits::NeedToConjugate), bool(RhsBlasTraits::NeedToConjugate)>::run + ( + lhs.rows(), // size + &lhs.coeffRef(0,0), lhs.outerStride(), // lhs info + actualRhsPtr, 1, // rhs info + actualDestPtr, // result info + actualAlpha // scale factor + ); + + if(!EvalToDest) + dest = MappedDest(actualDestPtr, dest.size()); + } +}; + +template<typename Lhs, typename Rhs, int RhsMode> +struct selfadjoint_product_impl<Lhs,0,true,Rhs,RhsMode,false> +{ + typedef typename Product<Lhs,Rhs>::Scalar Scalar; + enum { RhsUpLo = RhsMode&(Upper|Lower) }; + + template<typename Dest> + static void run(Dest& dest, const Lhs &a_lhs, const Rhs &a_rhs, const Scalar& alpha) + { + // let's simply transpose the product + Transpose<Dest> destT(dest); + selfadjoint_product_impl<Transpose<const Rhs>, int(RhsUpLo)==Upper ? Lower : Upper, false, + Transpose<const Lhs>, 0, true>::run(destT, a_rhs.transpose(), a_lhs.transpose(), alpha); + } +}; + +} // end namespace internal + +#endif // EIGEN_TEST_EVALUATORS + } // end namespace Eigen #endif // EIGEN_SELFADJOINT_MATRIX_VECTOR_H diff --git a/Eigen/src/Core/products/TriangularMatrixMatrix.h b/Eigen/src/Core/products/TriangularMatrixMatrix.h index db7b27f8e..fda6e2486 100644 --- a/Eigen/src/Core/products/TriangularMatrixMatrix.h +++ b/Eigen/src/Core/products/TriangularMatrixMatrix.h @@ -368,14 +368,16 @@ EIGEN_DONT_INLINE void product_triangular_matrix_matrix<Scalar,Index,Mode,false, /*************************************************************************** * Wrapper to product_triangular_matrix_matrix ***************************************************************************/ - +#ifndef EIGEN_TEST_EVALUATORS template<int Mode, bool LhsIsTriangular, typename Lhs, typename Rhs> struct traits<TriangularProduct<Mode,LhsIsTriangular,Lhs,false,Rhs,false> > : traits<ProductBase<TriangularProduct<Mode,LhsIsTriangular,Lhs,false,Rhs,false>, Lhs, Rhs> > {}; +#endif } // end namespace internal +#ifndef EIGEN_TEST_EVALUATORS template<int Mode, bool LhsIsTriangular, typename Lhs, typename Rhs> struct TriangularProduct<Mode,LhsIsTriangular,Lhs,false,Rhs,false> : public ProductBase<TriangularProduct<Mode,LhsIsTriangular,Lhs,false,Rhs,false>, Lhs, Rhs > @@ -417,6 +419,58 @@ struct TriangularProduct<Mode,LhsIsTriangular,Lhs,false,Rhs,false> ); } }; +#endif // EIGEN_TEST_EVALUATORS +#ifdef EIGEN_ENABLE_EVALUATORS +namespace internal { +template<int Mode, bool LhsIsTriangular, typename Lhs, typename Rhs> +struct triangular_product_impl<Mode,LhsIsTriangular,Lhs,false,Rhs,false> +{ + template<typename Dest> static void run(Dest& dst, const Lhs &a_lhs, const Rhs &a_rhs, const typename Dest::Scalar& alpha) + { + typedef typename Dest::Index Index; + typedef typename Dest::Scalar Scalar; + + typedef internal::blas_traits<Lhs> LhsBlasTraits; + typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; + typedef typename internal::remove_all<ActualLhsType>::type ActualLhsTypeCleaned; + typedef internal::blas_traits<Rhs> RhsBlasTraits; + typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; + typedef typename internal::remove_all<ActualRhsType>::type ActualRhsTypeCleaned; + + typename internal::add_const_on_value_type<ActualLhsType>::type lhs = LhsBlasTraits::extract(a_lhs); + typename internal::add_const_on_value_type<ActualRhsType>::type rhs = RhsBlasTraits::extract(a_rhs); + + Scalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(a_lhs) + * RhsBlasTraits::extractScalarFactor(a_rhs); + + typedef internal::gemm_blocking_space<(Dest::Flags&RowMajorBit) ? RowMajor : ColMajor,Scalar,Scalar, + Lhs::MaxRowsAtCompileTime, Rhs::MaxColsAtCompileTime, Lhs::MaxColsAtCompileTime,4> BlockingType; + + enum { IsLower = (Mode&Lower) == Lower }; + Index stripedRows = ((!LhsIsTriangular) || (IsLower)) ? lhs.rows() : (std::min)(lhs.rows(),lhs.cols()); + Index stripedCols = ((LhsIsTriangular) || (!IsLower)) ? rhs.cols() : (std::min)(rhs.cols(),rhs.rows()); + Index stripedDepth = LhsIsTriangular ? ((!IsLower) ? lhs.cols() : (std::min)(lhs.cols(),lhs.rows())) + : ((IsLower) ? rhs.rows() : (std::min)(rhs.rows(),rhs.cols())); + + BlockingType blocking(stripedRows, stripedCols, stripedDepth); + + internal::product_triangular_matrix_matrix<Scalar, Index, + Mode, LhsIsTriangular, + (internal::traits<ActualLhsTypeCleaned>::Flags&RowMajorBit) ? RowMajor : ColMajor, LhsBlasTraits::NeedToConjugate, + (internal::traits<ActualRhsTypeCleaned>::Flags&RowMajorBit) ? RowMajor : ColMajor, RhsBlasTraits::NeedToConjugate, + (internal::traits<Dest >::Flags&RowMajorBit) ? RowMajor : ColMajor> + ::run( + stripedRows, stripedCols, stripedDepth, // sizes + &lhs.coeffRef(0,0), lhs.outerStride(), // lhs info + &rhs.coeffRef(0,0), rhs.outerStride(), // rhs info + &dst.coeffRef(0,0), dst.outerStride(), // result info + actualAlpha, blocking + ); + } +}; + +} // end namespace internal +#endif // EIGEN_ENABLE_EVALUATORS } // end namespace Eigen diff --git a/Eigen/src/Core/products/TriangularMatrixVector.h b/Eigen/src/Core/products/TriangularMatrixVector.h index 817768481..19167c232 100644 --- a/Eigen/src/Core/products/TriangularMatrixVector.h +++ b/Eigen/src/Core/products/TriangularMatrixVector.h @@ -157,6 +157,8 @@ EIGEN_DONT_INLINE void triangular_matrix_vector_product<Index,Mode,LhsScalar,Con * Wrapper to product_triangular_vector ***************************************************************************/ +#ifndef EIGEN_TEST_EVALUATORS + template<int Mode, bool LhsIsTriangular, typename Lhs, typename Rhs> struct traits<TriangularProduct<Mode,LhsIsTriangular,Lhs,false,Rhs,true> > : traits<ProductBase<TriangularProduct<Mode,LhsIsTriangular,Lhs,false,Rhs,true>, Lhs, Rhs> > @@ -166,13 +168,14 @@ template<int Mode, bool LhsIsTriangular, typename Lhs, typename Rhs> struct traits<TriangularProduct<Mode,LhsIsTriangular,Lhs,true,Rhs,false> > : traits<ProductBase<TriangularProduct<Mode,LhsIsTriangular,Lhs,true,Rhs,false>, Lhs, Rhs> > {}; +#endif - -template<int StorageOrder> +template<int Mode,int StorageOrder> struct trmv_selector; } // end namespace internal +#ifndef EIGEN_TEST_EVALUATORS template<int Mode, typename Lhs, typename Rhs> struct TriangularProduct<Mode,true,Lhs,false,Rhs,true> : public ProductBase<TriangularProduct<Mode,true,Lhs,false,Rhs,true>, Lhs, Rhs > @@ -185,7 +188,7 @@ struct TriangularProduct<Mode,true,Lhs,false,Rhs,true> { eigen_assert(dst.rows()==m_lhs.rows() && dst.cols()==m_rhs.cols()); - internal::trmv_selector<(int(internal::traits<Lhs>::Flags)&RowMajorBit) ? RowMajor : ColMajor>::run(*this, dst, alpha); + internal::trmv_selector<Mode,(int(internal::traits<Lhs>::Flags)&RowMajorBit) ? RowMajor : ColMajor>::run(m_lhs, m_rhs, dst, alpha); } }; @@ -201,39 +204,71 @@ struct TriangularProduct<Mode,false,Lhs,true,Rhs,false> { eigen_assert(dst.rows()==m_lhs.rows() && dst.cols()==m_rhs.cols()); - typedef TriangularProduct<(Mode & (UnitDiag|ZeroDiag)) | ((Mode & Lower) ? Upper : Lower),true,Transpose<const Rhs>,false,Transpose<const Lhs>,true> TriangularProductTranspose; Transpose<Dest> dstT(dst); - internal::trmv_selector<(int(internal::traits<Rhs>::Flags)&RowMajorBit) ? ColMajor : RowMajor>::run( - TriangularProductTranspose(m_rhs.transpose(),m_lhs.transpose()), dstT, alpha); + internal::trmv_selector<(Mode & (UnitDiag|ZeroDiag)) | ((Mode & Lower) ? Upper : Lower), + (int(internal::traits<Rhs>::Flags)&RowMajorBit) ? ColMajor : RowMajor> + ::run(m_rhs.transpose(),m_lhs.transpose(), dstT, alpha); + } +}; + +#else // EIGEN_TEST_EVALUATORS +namespace internal { + +template<int Mode, typename Lhs, typename Rhs> +struct triangular_product_impl<Mode,true,Lhs,false,Rhs,true> +{ + template<typename Dest> static void run(Dest& dst, const Lhs &lhs, const Rhs &rhs, const typename Dest::Scalar& alpha) + { + eigen_assert(dst.rows()==lhs.rows() && dst.cols()==rhs.cols()); + + internal::trmv_selector<Mode,(int(internal::traits<Lhs>::Flags)&RowMajorBit) ? RowMajor : ColMajor>::run(lhs, rhs, dst, alpha); + } +}; + +template<int Mode, typename Lhs, typename Rhs> +struct triangular_product_impl<Mode,false,Lhs,true,Rhs,false> +{ + template<typename Dest> static void run(Dest& dst, const Lhs &lhs, const Rhs &rhs, const typename Dest::Scalar& alpha) + { + eigen_assert(dst.rows()==lhs.rows() && dst.cols()==rhs.cols()); + + Transpose<Dest> dstT(dst); + internal::trmv_selector<(Mode & (UnitDiag|ZeroDiag)) | ((Mode & Lower) ? Upper : Lower), + (int(internal::traits<Rhs>::Flags)&RowMajorBit) ? ColMajor : RowMajor> + ::run(rhs.transpose(),lhs.transpose(), dstT, alpha); } }; +} // end namespace internal +#endif // EIGEN_TEST_EVALUATORS + namespace internal { // TODO: find a way to factorize this piece of code with gemv_selector since the logic is exactly the same. -template<> struct trmv_selector<ColMajor> +template<int Mode> struct trmv_selector<Mode,ColMajor> { - template<int Mode, typename Lhs, typename Rhs, typename Dest> - static void run(const TriangularProduct<Mode,true,Lhs,false,Rhs,true>& prod, Dest& dest, const typename TriangularProduct<Mode,true,Lhs,false,Rhs,true>::Scalar& alpha) + template<typename Lhs, typename Rhs, typename Dest> + static void run(const Lhs &lhs, const Rhs &rhs, Dest& dest, const typename Dest::Scalar& alpha) { - typedef TriangularProduct<Mode,true,Lhs,false,Rhs,true> ProductType; - typedef typename ProductType::Index Index; - typedef typename ProductType::LhsScalar LhsScalar; - typedef typename ProductType::RhsScalar RhsScalar; - typedef typename ProductType::Scalar ResScalar; - typedef typename ProductType::RealScalar RealScalar; - typedef typename ProductType::ActualLhsType ActualLhsType; - typedef typename ProductType::ActualRhsType ActualRhsType; - typedef typename ProductType::LhsBlasTraits LhsBlasTraits; - typedef typename ProductType::RhsBlasTraits RhsBlasTraits; + typedef typename Dest::Index Index; + typedef typename Lhs::Scalar LhsScalar; + typedef typename Rhs::Scalar RhsScalar; + typedef typename Dest::Scalar ResScalar; + typedef typename Dest::RealScalar RealScalar; + + typedef internal::blas_traits<Lhs> LhsBlasTraits; + typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; + typedef internal::blas_traits<Rhs> RhsBlasTraits; + typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; + typedef Map<Matrix<ResScalar,Dynamic,1>, Aligned> MappedDest; - typename internal::add_const_on_value_type<ActualLhsType>::type actualLhs = LhsBlasTraits::extract(prod.lhs()); - typename internal::add_const_on_value_type<ActualRhsType>::type actualRhs = RhsBlasTraits::extract(prod.rhs()); + typename internal::add_const_on_value_type<ActualLhsType>::type actualLhs = LhsBlasTraits::extract(lhs); + typename internal::add_const_on_value_type<ActualRhsType>::type actualRhs = RhsBlasTraits::extract(rhs); - ResScalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(prod.lhs()) - * RhsBlasTraits::extractScalarFactor(prod.rhs()); + ResScalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(lhs) + * RhsBlasTraits::extractScalarFactor(rhs); enum { // FIXME find a way to allow an inner stride on the result if packet_traits<Scalar>::size==1 @@ -288,33 +323,33 @@ template<> struct trmv_selector<ColMajor> } }; -template<> struct trmv_selector<RowMajor> +template<int Mode> struct trmv_selector<Mode,RowMajor> { - template<int Mode, typename Lhs, typename Rhs, typename Dest> - static void run(const TriangularProduct<Mode,true,Lhs,false,Rhs,true>& prod, Dest& dest, const typename TriangularProduct<Mode,true,Lhs,false,Rhs,true>::Scalar& alpha) + template<typename Lhs, typename Rhs, typename Dest> + static void run(const Lhs &lhs, const Rhs &rhs, Dest& dest, const typename Dest::Scalar& alpha) { - typedef TriangularProduct<Mode,true,Lhs,false,Rhs,true> ProductType; - typedef typename ProductType::LhsScalar LhsScalar; - typedef typename ProductType::RhsScalar RhsScalar; - typedef typename ProductType::Scalar ResScalar; - typedef typename ProductType::Index Index; - typedef typename ProductType::ActualLhsType ActualLhsType; - typedef typename ProductType::ActualRhsType ActualRhsType; - typedef typename ProductType::_ActualRhsType _ActualRhsType; - typedef typename ProductType::LhsBlasTraits LhsBlasTraits; - typedef typename ProductType::RhsBlasTraits RhsBlasTraits; - - typename add_const<ActualLhsType>::type actualLhs = LhsBlasTraits::extract(prod.lhs()); - typename add_const<ActualRhsType>::type actualRhs = RhsBlasTraits::extract(prod.rhs()); - - ResScalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(prod.lhs()) - * RhsBlasTraits::extractScalarFactor(prod.rhs()); + typedef typename Dest::Index Index; + typedef typename Lhs::Scalar LhsScalar; + typedef typename Rhs::Scalar RhsScalar; + typedef typename Dest::Scalar ResScalar; + + typedef internal::blas_traits<Lhs> LhsBlasTraits; + typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType; + typedef internal::blas_traits<Rhs> RhsBlasTraits; + typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType; + typedef typename internal::remove_all<ActualRhsType>::type ActualRhsTypeCleaned; + + typename add_const<ActualLhsType>::type actualLhs = LhsBlasTraits::extract(lhs); + typename add_const<ActualRhsType>::type actualRhs = RhsBlasTraits::extract(rhs); + + ResScalar actualAlpha = alpha * LhsBlasTraits::extractScalarFactor(lhs) + * RhsBlasTraits::extractScalarFactor(rhs); enum { - DirectlyUseRhs = _ActualRhsType::InnerStrideAtCompileTime==1 + DirectlyUseRhs = ActualRhsTypeCleaned::InnerStrideAtCompileTime==1 }; - gemv_static_vector_if<RhsScalar,_ActualRhsType::SizeAtCompileTime,_ActualRhsType::MaxSizeAtCompileTime,!DirectlyUseRhs> static_rhs; + gemv_static_vector_if<RhsScalar,ActualRhsTypeCleaned::SizeAtCompileTime,ActualRhsTypeCleaned::MaxSizeAtCompileTime,!DirectlyUseRhs> static_rhs; ei_declare_aligned_stack_constructed_variable(RhsScalar,actualRhsPtr,actualRhs.size(), DirectlyUseRhs ? const_cast<RhsScalar*>(actualRhs.data()) : static_rhs.data()); @@ -325,7 +360,7 @@ template<> struct trmv_selector<RowMajor> Index size = actualRhs.size(); EIGEN_DENSE_STORAGE_CTOR_PLUGIN #endif - Map<typename _ActualRhsType::PlainObject>(actualRhsPtr, actualRhs.size()) = actualRhs; + Map<typename ActualRhsTypeCleaned::PlainObject>(actualRhsPtr, actualRhs.size()) = actualRhs; } internal::triangular_matrix_vector_product diff --git a/Eigen/src/Core/util/Constants.h b/Eigen/src/Core/util/Constants.h index 05107fdfe..86817a989 100644 --- a/Eigen/src/Core/util/Constants.h +++ b/Eigen/src/Core/util/Constants.h @@ -53,14 +53,13 @@ const int Infinity = -1; const unsigned int RowMajorBit = 0x1; /** \ingroup flags - * * means the expression should be evaluated by the calling expression */ const unsigned int EvalBeforeNestingBit = 0x2; /** \ingroup flags - * + * \deprecated * means the expression should be evaluated before any assignment */ -const unsigned int EvalBeforeAssigningBit = 0x4; +const unsigned int EvalBeforeAssigningBit = 0x4; // FIXME deprecated /** \ingroup flags * @@ -155,6 +154,16 @@ const unsigned int AlignedBit = 0x80; const unsigned int NestByRefBit = 0x100; +/** \ingroup flags + * + * for an expression, this means that the storage order + * can be either row-major or column-major. + * The precise choice will be decided at evaluation time or when + * combined with other expressions. + * \sa \ref RowMajorBit, \ref TopicStorageOrders */ +const unsigned int NoPreferredStorageOrderBit = 0x200; + + // list of flags that are inherited by default const unsigned int HereditaryBits = RowMajorBit | EvalBeforeNestingBit @@ -425,7 +434,7 @@ namespace Architecture /** \internal \ingroup enums * Enum used as template parameter in GeneralProduct. */ -enum { CoeffBasedProductMode, LazyCoeffBasedProductMode, OuterProduct, InnerProduct, GemvProduct, GemmProduct }; +enum { DefaultProduct=0, CoeffBasedProductMode, LazyCoeffBasedProductMode, LazyProduct, OuterProduct, InnerProduct, GemvProduct, GemmProduct }; /** \internal \ingroup enums * Enum used in experimental parallel implementation. */ @@ -434,12 +443,25 @@ enum Action {GetAction, SetAction}; /** The type used to identify a dense storage. */ struct Dense {}; +/** The type used to identify a permutation storage. */ +struct PermutationStorage {}; + /** The type used to identify a matrix expression */ struct MatrixXpr {}; /** The type used to identify an array expression */ struct ArrayXpr {}; +// An evaluator must define its shape. By default, it can be one of the following: +struct DenseShape { static std::string debugName() { return "DenseShape"; } }; +struct HomogeneousShape { static std::string debugName() { return "HomogeneousShape"; } }; +struct DiagonalShape { static std::string debugName() { return "DiagonalShape"; } }; +struct BandShape { static std::string debugName() { return "BandShape"; } }; +struct TriangularShape { static std::string debugName() { return "TriangularShape"; } }; +struct SelfAdjointShape { static std::string debugName() { return "SelfAdjointShape"; } }; +struct PermutationShape { static std::string debugName() { return "PermutationShape"; } }; +struct SparseShape { static std::string debugName() { return "SparseShape"; } }; + } // end namespace Eigen #endif // EIGEN_CONSTANTS_H diff --git a/Eigen/src/Core/util/ForwardDeclarations.h b/Eigen/src/Core/util/ForwardDeclarations.h index 33deb88ec..99aa9b372 100644 --- a/Eigen/src/Core/util/ForwardDeclarations.h +++ b/Eigen/src/Core/util/ForwardDeclarations.h @@ -36,6 +36,10 @@ template<typename Derived> struct accessors_level }; }; +template<typename T> struct evaluator_traits; + +template< typename T> struct evaluator; + } // end namespace internal template<typename T> struct NumTraits; @@ -87,11 +91,19 @@ template<typename NullaryOp, typename MatrixType> class CwiseNullaryOp; template<typename UnaryOp, typename MatrixType> class CwiseUnaryOp; template<typename ViewOp, typename MatrixType> class CwiseUnaryView; template<typename BinaryOp, typename Lhs, typename Rhs> class CwiseBinaryOp; -template<typename BinOp, typename Lhs, typename Rhs> class SelfCwiseBinaryOp; -template<typename Derived, typename Lhs, typename Rhs> class ProductBase; -template<typename Lhs, typename Rhs> class Product; -template<typename Lhs, typename Rhs, int Mode> class GeneralProduct; -template<typename Lhs, typename Rhs, int NestingFlags> class CoeffBasedProduct; +template<typename BinOp, typename Lhs, typename Rhs> class SelfCwiseBinaryOp; // TODO deprecated +template<typename Derived, typename Lhs, typename Rhs> class ProductBase; // TODO deprecated +template<typename Decomposition, typename Rhstype> class Solve; +template<typename XprType> class Inverse; + +namespace internal { + template<typename Lhs, typename Rhs> struct product_tag; +} + +template<typename Lhs, typename Rhs, int Option = DefaultProduct> class Product; + +template<typename Lhs, typename Rhs, int Mode> class GeneralProduct; // TODO deprecated +template<typename Lhs, typename Rhs, int NestingFlags> class CoeffBasedProduct; // TODO deprecated template<typename Derived> class DiagonalBase; template<typename _DiagonalVectorType> class DiagonalWrapper; @@ -109,7 +121,12 @@ template<typename Derived, int Level = internal::accessors_level<Derived>::has_write_access ? WriteAccessors : ReadOnlyAccessors > class MapBase; template<int InnerStrideAtCompileTime, int OuterStrideAtCompileTime> class Stride; +template<int Value = Dynamic> class InnerStride; +template<int Value = Dynamic> class OuterStride; template<typename MatrixType, int MapOptions=Unaligned, typename StrideType = Stride<0,0> > class Map; +template<typename Derived> class RefBase; +template<typename PlainObjectType, int Options = 0, + typename StrideType = typename internal::conditional<PlainObjectType::IsVectorAtCompileTime,InnerStride<1>,OuterStride<> >::type > class Ref; template<typename Derived> class TriangularBase; template<typename MatrixType, unsigned int Mode> class TriangularView; @@ -122,8 +139,10 @@ template<typename ExpressionType> class ArrayWrapper; template<typename ExpressionType> class MatrixWrapper; namespace internal { +#ifndef EIGEN_TEST_EVALUATROS template<typename DecompositionType, typename Rhs> struct solve_retval_base; template<typename DecompositionType, typename Rhs> struct solve_retval; +#endif template<typename DecompositionType> struct kernel_retval_base; template<typename DecompositionType> struct kernel_retval; template<typename DecompositionType> struct image_retval_base; @@ -136,6 +155,18 @@ template<typename _Scalar, int Rows=Dynamic, int Cols=Dynamic, int Supers=Dynami namespace internal { template<typename Lhs, typename Rhs> struct product_type; +/** \internal + * \class product_evaluator + * Products need their own evaluator with more template arguments allowing for + * easier partial template specializations. + */ +template< typename T, + int ProductTag = internal::product_type<typename T::Lhs,typename T::Rhs>::ret, + typename LhsShape = typename evaluator_traits<typename T::Lhs>::Shape, + typename RhsShape = typename evaluator_traits<typename T::Rhs>::Shape, + typename LhsScalar = typename traits<typename T::Lhs>::Scalar, + typename RhsScalar = typename traits<typename T::Rhs>::Scalar + > struct product_evaluator; } template<typename Lhs, typename Rhs, diff --git a/Eigen/src/Core/util/Macros.h b/Eigen/src/Core/util/Macros.h index 5e9b0a112..d029e0c6c 100644 --- a/Eigen/src/Core/util/Macros.h +++ b/Eigen/src/Core/util/Macros.h @@ -367,6 +367,8 @@ namespace Eigen { * documentation in a single line. **/ +#ifndef EIGEN_TEST_EVALUATORS + #define EIGEN_GENERIC_PUBLIC_INTERFACE(Derived) \ typedef typename Eigen::internal::traits<Derived>::Scalar Scalar; /*!< \brief Numeric type, e.g. float, double, int or std::complex<float>. */ \ typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; /*!< \brief The underlying numeric type for composed scalar types. \details In cases where Scalar is e.g. std::complex<T>, T were corresponding to RealScalar. */ \ @@ -403,7 +405,46 @@ namespace Eigen { using Base::derived; \ using Base::const_cast_derived; +#else + +// TODO The EIGEN_DENSE_PUBLIC_INTERFACE should not exists anymore + +#define EIGEN_GENERIC_PUBLIC_INTERFACE(Derived) \ + typedef typename Eigen::internal::traits<Derived>::Scalar Scalar; /*!< \brief Numeric type, e.g. float, double, int or std::complex<float>. */ \ + typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; /*!< \brief The underlying numeric type for composed scalar types. \details In cases where Scalar is e.g. std::complex<T>, T were corresponding to RealScalar. */ \ + typedef typename Base::CoeffReturnType CoeffReturnType; /*!< \brief The return type for coefficient access. \details Depending on whether the object allows direct coefficient access (e.g. for a MatrixXd), this type is either 'const Scalar&' or simply 'Scalar' for objects that do not allow direct coefficient access. */ \ + typedef typename Eigen::internal::nested<Derived>::type Nested; \ + typedef typename Eigen::internal::traits<Derived>::StorageKind StorageKind; \ + typedef typename Eigen::internal::traits<Derived>::Index Index; \ + enum { RowsAtCompileTime = Eigen::internal::traits<Derived>::RowsAtCompileTime, \ + ColsAtCompileTime = Eigen::internal::traits<Derived>::ColsAtCompileTime, \ + Flags = Eigen::internal::traits<Derived>::Flags, \ + SizeAtCompileTime = Base::SizeAtCompileTime, \ + MaxSizeAtCompileTime = Base::MaxSizeAtCompileTime, \ + IsVectorAtCompileTime = Base::IsVectorAtCompileTime }; + + +#define EIGEN_DENSE_PUBLIC_INTERFACE(Derived) \ + typedef typename Eigen::internal::traits<Derived>::Scalar Scalar; /*!< \brief Numeric type, e.g. float, double, int or std::complex<float>. */ \ + typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; /*!< \brief The underlying numeric type for composed scalar types. \details In cases where Scalar is e.g. std::complex<T>, T were corresponding to RealScalar. */ \ + typedef typename Base::PacketScalar PacketScalar; \ + typedef typename Base::CoeffReturnType CoeffReturnType; /*!< \brief The return type for coefficient access. \details Depending on whether the object allows direct coefficient access (e.g. for a MatrixXd), this type is either 'const Scalar&' or simply 'Scalar' for objects that do not allow direct coefficient access. */ \ + typedef typename Eigen::internal::nested<Derived>::type Nested; \ + typedef typename Eigen::internal::traits<Derived>::StorageKind StorageKind; \ + typedef typename Eigen::internal::traits<Derived>::Index Index; \ + enum { RowsAtCompileTime = Eigen::internal::traits<Derived>::RowsAtCompileTime, \ + ColsAtCompileTime = Eigen::internal::traits<Derived>::ColsAtCompileTime, \ + MaxRowsAtCompileTime = Eigen::internal::traits<Derived>::MaxRowsAtCompileTime, \ + MaxColsAtCompileTime = Eigen::internal::traits<Derived>::MaxColsAtCompileTime, \ + Flags = Eigen::internal::traits<Derived>::Flags, \ + SizeAtCompileTime = Base::SizeAtCompileTime, \ + MaxSizeAtCompileTime = Base::MaxSizeAtCompileTime, \ + IsVectorAtCompileTime = Base::IsVectorAtCompileTime }; \ + using Base::derived; \ + using Base::const_cast_derived; +#endif // EIGEN_TEST_EVALUATORS + #define EIGEN_PLAIN_ENUM_MIN(a,b) (((int)a <= (int)b) ? (int)a : (int)b) #define EIGEN_PLAIN_ENUM_MAX(a,b) (((int)a >= (int)b) ? (int)a : (int)b) diff --git a/Eigen/src/Core/util/Meta.h b/Eigen/src/Core/util/Meta.h index b99b8849e..f3bafd5af 100644 --- a/Eigen/src/Core/util/Meta.h +++ b/Eigen/src/Core/util/Meta.h @@ -274,18 +274,6 @@ template<typename T> struct scalar_product_traits<std::complex<T>, T> // typedef typename scalar_product_traits<typename remove_all<ArgType0>::type, typename remove_all<ArgType1>::type>::ReturnType type; // }; -template<typename T> struct is_diagonal -{ enum { ret = false }; }; - -template<typename T> struct is_diagonal<DiagonalBase<T> > -{ enum { ret = true }; }; - -template<typename T> struct is_diagonal<DiagonalWrapper<T> > -{ enum { ret = true }; }; - -template<typename T, int S> struct is_diagonal<DiagonalMatrix<T,S> > -{ enum { ret = true }; }; - } // end namespace internal namespace numext { diff --git a/Eigen/src/Core/util/StaticAssert.h b/Eigen/src/Core/util/StaticAssert.h index 59aa0811c..54a16ebf2 100644 --- a/Eigen/src/Core/util/StaticAssert.h +++ b/Eigen/src/Core/util/StaticAssert.h @@ -84,13 +84,15 @@ THIS_EXPRESSION_IS_NOT_A_LVALUE__IT_IS_READ_ONLY, YOU_ARE_TRYING_TO_USE_AN_INDEX_BASED_ACCESSOR_ON_AN_EXPRESSION_THAT_DOES_NOT_SUPPORT_THAT, THIS_METHOD_IS_ONLY_FOR_1x1_EXPRESSIONS, + THIS_METHOD_IS_ONLY_FOR_INNER_OR_LAZY_PRODUCTS, THIS_METHOD_IS_ONLY_FOR_EXPRESSIONS_OF_BOOL, THIS_METHOD_IS_ONLY_FOR_ARRAYS_NOT_MATRICES, YOU_PASSED_A_ROW_VECTOR_BUT_A_COLUMN_VECTOR_WAS_EXPECTED, YOU_PASSED_A_COLUMN_VECTOR_BUT_A_ROW_VECTOR_WAS_EXPECTED, THE_INDEX_TYPE_MUST_BE_A_SIGNED_TYPE, THE_STORAGE_ORDER_OF_BOTH_SIDES_MUST_MATCH, - OBJECT_ALLOCATED_ON_STACK_IS_TOO_BIG + OBJECT_ALLOCATED_ON_STACK_IS_TOO_BIG, + IMPLICIT_CONVERSION_TO_SCALAR_IS_FOR_INNER_PRODUCT_ONLY }; }; @@ -157,7 +159,7 @@ #define EIGEN_PREDICATE_SAME_MATRIX_SIZE(TYPE0,TYPE1) \ ( \ - (int(TYPE0::SizeAtCompileTime)==0 && int(TYPE1::SizeAtCompileTime)==0) \ + (int(internal::size_of_xpr_at_compile_time<TYPE0>::ret)==0 && int(internal::size_of_xpr_at_compile_time<TYPE1>::ret)==0) \ || (\ (int(TYPE0::RowsAtCompileTime)==Eigen::Dynamic \ || int(TYPE1::RowsAtCompileTime)==Eigen::Dynamic \ diff --git a/Eigen/src/Core/util/XprHelper.h b/Eigen/src/Core/util/XprHelper.h index 1b3e122e1..5ed3b39b7 100644 --- a/Eigen/src/Core/util/XprHelper.h +++ b/Eigen/src/Core/util/XprHelper.h @@ -125,6 +125,7 @@ template<typename _Scalar, int _Rows, int _Cols, typedef Matrix<_Scalar, _Rows, _Cols, Options, _MaxRows, _MaxCols> type; }; +#ifndef EIGEN_TEST_EVALUATORS template<typename Scalar, int Rows, int Cols, int Options, int MaxRows, int MaxCols> class compute_matrix_flags { @@ -159,11 +160,67 @@ class compute_matrix_flags enum { ret = LinearAccessBit | LvalueBit | DirectAccessBit | NestByRefBit | packet_access_bit | row_major_bit | aligned_bit }; }; +#else // EIGEN_TEST_EVALUATORS + +template<typename Scalar, int Rows, int Cols, int Options, int MaxRows, int MaxCols> +class compute_matrix_flags +{ + enum { row_major_bit = Options&RowMajor ? RowMajorBit : 0 }; + public: + // FIXME currently we still have to handle DirectAccessBit at the expression level to handle DenseCoeffsBase<> + // and then propagate this information to the evaluator's flags. + // However, I (Gael) think that DirectAccessBit should only matter at the evaluation stage. + enum { ret = DirectAccessBit | LvalueBit | NestByRefBit | row_major_bit }; +}; +#endif + +#ifdef EIGEN_ENABLE_EVALUATORS +template<typename Scalar, int Rows, int Cols, int Options, int MaxRows, int MaxCols> +class compute_matrix_evaluator_flags +{ + enum { + row_major_bit = Options&RowMajor ? RowMajorBit : 0, + is_dynamic_size_storage = MaxRows==Dynamic || MaxCols==Dynamic, + + aligned_bit = + ( + ((Options&DontAlign)==0) + && ( +#if EIGEN_ALIGN_STATICALLY + ((!is_dynamic_size_storage) && (((MaxCols*MaxRows*int(sizeof(Scalar))) % EIGEN_ALIGN_BYTES) == 0)) +#else + 0 +#endif + + || + +#if EIGEN_ALIGN + is_dynamic_size_storage +#else + 0 +#endif + + ) + ) ? AlignedBit : 0, + packet_access_bit = packet_traits<Scalar>::Vectorizable && aligned_bit ? PacketAccessBit : 0 + }; + + public: + enum { ret = LinearAccessBit | DirectAccessBit | packet_access_bit | row_major_bit | aligned_bit }; +}; + +#endif // EIGEN_ENABLE_EVALUATORS + template<int _Rows, int _Cols> struct size_at_compile_time { enum { ret = (_Rows==Dynamic || _Cols==Dynamic) ? Dynamic : _Rows * _Cols }; }; +template<typename XprType> struct size_of_xpr_at_compile_time +{ + enum { ret = size_at_compile_time<traits<XprType>::RowsAtCompileTime,traits<XprType>::ColsAtCompileTime>::ret }; +}; + /* plain_matrix_type : the difference from eval is that plain_matrix_type is always a plain matrix type, * whereas eval is a const reference in the case of a matrix */ @@ -174,6 +231,10 @@ template<typename T> struct plain_matrix_type<T,Dense> { typedef typename plain_matrix_type_dense<T,typename traits<T>::XprKind>::type type; }; +template<typename T> struct plain_matrix_type<T,DiagonalShape> +{ + typedef typename T::PlainObject type; +}; template<typename T> struct plain_matrix_type_dense<T,MatrixXpr> { @@ -216,6 +277,11 @@ template<typename T> struct eval<T,Dense> // > type; }; +template<typename T> struct eval<T,DiagonalShape> +{ + typedef typename plain_matrix_type<T>::type type; +}; + // for matrices, no need to evaluate, just use a const reference to avoid a useless copy template<typename _Scalar, int _Rows, int _Cols, int _Options, int _MaxRows, int _MaxCols> struct eval<Matrix<_Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols>, Dense> @@ -294,6 +360,10 @@ struct transfer_constness >::type type; }; + + +#ifndef EIGEN_TEST_EVALUATORS + /** \internal Determines how a given expression should be nested into another one. * For example, when you do a * (b+c), Eigen will determine how the expression b+c should be * nested into the bigger product expression. The choice is between nesting the expression b+c as-is, or @@ -341,6 +411,50 @@ template<typename T, int n=1, typename PlainObject = typename eval<T>::type> str >::type type; }; +#else + +// When using evaluators, we never evaluate when assembling the expression!! +// TODO: get rid of this nested class since it's just an alias for ref_selector. +template<typename T, int n=1, typename PlainObject = void> struct nested +{ + typedef typename ref_selector<T>::type type; +}; + +#endif // EIGEN_TEST_EVALUATORS + +#ifdef EIGEN_ENABLE_EVALUATORS +// However, we still need a mechanism to detect whether an expression which is evaluated multiple time +// has to be evaluated into a temporary. +// That's the purpose of this new nested_eval helper: +template<typename T, int n, typename PlainObject = typename eval<T>::type> struct nested_eval +{ + enum { + // For the purpose of this test, to keep it reasonably simple, we arbitrarily choose a value of Dynamic values. + // the choice of 10000 makes it larger than any practical fixed value and even most dynamic values. + // in extreme cases where these assumptions would be wrong, we would still at worst suffer performance issues + // (poor choice of temporaries). + // It's important that this value can still be squared without integer overflowing. + DynamicAsInteger = 10000, + ScalarReadCost = NumTraits<typename traits<T>::Scalar>::ReadCost, + ScalarReadCostAsInteger = ScalarReadCost == Dynamic ? int(DynamicAsInteger) : int(ScalarReadCost), + CoeffReadCost = evaluator<T>::CoeffReadCost, // TODO What if an evaluator evaluate itself into a tempory? + // Then CoeffReadCost will be small but we still have to evaluate if n>1... + // The solution might be to ask the evaluator if it creates a temp. Perhaps we could even ask the number of temps? + CoeffReadCostAsInteger = CoeffReadCost == Dynamic ? int(DynamicAsInteger) : int(CoeffReadCost), + NAsInteger = n == Dynamic ? int(DynamicAsInteger) : n, + CostEvalAsInteger = (NAsInteger+1) * ScalarReadCostAsInteger + CoeffReadCostAsInteger, + CostNoEvalAsInteger = NAsInteger * CoeffReadCostAsInteger + }; + + typedef typename conditional< + ( (int(evaluator<T>::Flags) & EvalBeforeNestingBit) || + (int(CostEvalAsInteger) < int(CostNoEvalAsInteger)) ), + PlainObject, + typename ref_selector<T>::type + >::type type; +}; +#endif + template<typename T> EIGEN_DEVICE_FUNC T* const_cast_ptr(const T* ptr) @@ -366,6 +480,15 @@ struct dense_xpr_base<Derived, ArrayXpr> typedef ArrayBase<Derived> type; }; +template<typename Derived, typename XprKind = typename traits<Derived>::XprKind, typename StorageKind = typename traits<Derived>::StorageKind> +struct generic_xpr_base; + +template<typename Derived, typename XprKind> +struct generic_xpr_base<Derived, XprKind, Dense> +{ + typedef typename dense_xpr_base<Derived,XprKind>::type type; +}; + /** \internal Helper base class to add a scalar multiple operator * overloads for complex types */ template<typename Derived,typename Scalar,typename OtherScalar, @@ -401,12 +524,59 @@ template<typename XprType, typename CastType> struct cast_return_type const XprType&,CastType>::type type; }; -template <typename A, typename B> struct promote_storage_type; +/** \internal Specify the "storage kind" of applying a coefficient-wise + * binary operations between two expressions of kinds A and B respectively. + * The template parameter Functor permits to specialize the resulting storage kind wrt to + * the functor. + * The default rules are as follows: + * \code + * A op A -> A + * A op dense -> dense + * dense op B -> dense + * A * dense -> A + * dense * B -> B + * \endcode + */ +template <typename A, typename B, typename Functor> struct cwise_promote_storage_type; + +template <typename A, typename Functor> struct cwise_promote_storage_type<A,A,Functor> { typedef A ret; }; +template <typename Functor> struct cwise_promote_storage_type<Dense,Dense,Functor> { typedef Dense ret; }; +template <typename ScalarA, typename ScalarB> struct cwise_promote_storage_type<Dense,Dense,scalar_product_op<ScalarA,ScalarB> > { typedef Dense ret; }; +template <typename A, typename Functor> struct cwise_promote_storage_type<A,Dense,Functor> { typedef Dense ret; }; +template <typename B, typename Functor> struct cwise_promote_storage_type<Dense,B,Functor> { typedef Dense ret; }; +template <typename A, typename ScalarA, typename ScalarB> struct cwise_promote_storage_type<A,Dense,scalar_product_op<ScalarA,ScalarB> > { typedef A ret; }; +template <typename B, typename ScalarA, typename ScalarB> struct cwise_promote_storage_type<Dense,B,scalar_product_op<ScalarA,ScalarB> > { typedef B ret; }; + +/** \internal Specify the "storage kind" of multiplying an expression of kind A with kind B. + * The template parameter ProductTag permits to specialize the resulting storage kind wrt to + * some compile-time properties of the product: GemmProduct, GemvProduct, OuterProduct, InnerProduct. + * The default rules are as follows: + * \code + * K * K -> K + * dense * K -> dense + * K * dense -> dense + * diag * K -> K + * K * diag -> K + * Perm * K -> K + * K * Perm -> K + * \endcode + */ +template <typename A, typename B, int ProductTag> struct product_promote_storage_type; -template <typename A> struct promote_storage_type<A,A> -{ - typedef A ret; -}; +template <typename A, int ProductTag> struct product_promote_storage_type<A, A, ProductTag> { typedef A ret;}; +template <int ProductTag> struct product_promote_storage_type<Dense, Dense, ProductTag> { typedef Dense ret;}; +template <typename A, int ProductTag> struct product_promote_storage_type<A, Dense, ProductTag> { typedef Dense ret; }; +template <typename B, int ProductTag> struct product_promote_storage_type<Dense, B, ProductTag> { typedef Dense ret; }; + +template <typename A, int ProductTag> struct product_promote_storage_type<A, DiagonalShape, ProductTag> { typedef A ret; }; +template <typename B, int ProductTag> struct product_promote_storage_type<DiagonalShape, B, ProductTag> { typedef B ret; }; +template <int ProductTag> struct product_promote_storage_type<Dense, DiagonalShape, ProductTag> { typedef Dense ret; }; +template <int ProductTag> struct product_promote_storage_type<DiagonalShape, Dense, ProductTag> { typedef Dense ret; }; + +template <typename A, int ProductTag> struct product_promote_storage_type<A, PermutationStorage, ProductTag> { typedef A ret; }; +template <typename B, int ProductTag> struct product_promote_storage_type<PermutationStorage, B, ProductTag> { typedef B ret; }; +template <int ProductTag> struct product_promote_storage_type<Dense, PermutationStorage, ProductTag> { typedef Dense ret; }; +template <int ProductTag> struct product_promote_storage_type<PermutationStorage, Dense, ProductTag> { typedef Dense ret; }; /** \internal gives the plain matrix or array type to store a row/column/diagonal of a matrix type. * \param Scalar optional parameter allowing to pass a different scalar type than the one of the MatrixType. @@ -464,8 +634,36 @@ struct is_lvalue bool(traits<ExpressionType>::Flags & LvalueBit) }; }; +template<typename T> struct is_diagonal +{ enum { ret = false }; }; + +template<typename T> struct is_diagonal<DiagonalBase<T> > +{ enum { ret = true }; }; + +template<typename T> struct is_diagonal<DiagonalWrapper<T> > +{ enum { ret = true }; }; + +template<typename T, int S> struct is_diagonal<DiagonalMatrix<T,S> > +{ enum { ret = true }; }; + +template<typename S1, typename S2> struct glue_shapes; +template<> struct glue_shapes<DenseShape,TriangularShape> { typedef TriangularShape type; }; + } // end namespace internal +// we require Lhs and Rhs to have the same scalar type. Currently there is no example of a binary functor +// that would take two operands of different types. If there were such an example, then this check should be +// moved to the BinaryOp functors, on a per-case basis. This would however require a change in the BinaryOp functors, as +// currently they take only one typename Scalar template parameter. +// It is tempting to always allow mixing different types but remember that this is often impossible in the vectorized paths. +// So allowing mixing different types gives very unexpected errors when enabling vectorization, when the user tries to +// add together a float matrix and a double matrix. +#define EIGEN_CHECK_BINARY_COMPATIBILIY(BINOP,LHS,RHS) \ + EIGEN_STATIC_ASSERT((internal::functor_is_product_like<BINOP>::ret \ + ? int(internal::scalar_product_traits<LHS, RHS>::Defined) \ + : int(internal::is_same<LHS, RHS>::value)), \ + YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) + } // end namespace Eigen #endif // EIGEN_XPRHELPER_H diff --git a/Eigen/src/Eigenvalues/Tridiagonalization.h b/Eigen/src/Eigenvalues/Tridiagonalization.h index 192278d68..e3a27f275 100644 --- a/Eigen/src/Eigenvalues/Tridiagonalization.h +++ b/Eigen/src/Eigenvalues/Tridiagonalization.h @@ -18,8 +18,10 @@ namespace internal { template<typename MatrixType> struct TridiagonalizationMatrixTReturnType; template<typename MatrixType> struct traits<TridiagonalizationMatrixTReturnType<MatrixType> > + : public traits<typename MatrixType::PlainObject> { - typedef typename MatrixType::PlainObject ReturnType; + typedef typename MatrixType::PlainObject ReturnType; // FIXME shall it be a BandMatrix? + enum { Flags = 0 }; }; template<typename MatrixType, typename CoeffVectorType> diff --git a/Eigen/src/Geometry/AlignedBox.h b/Eigen/src/Geometry/AlignedBox.h index b6a2f0e24..1d1daaa61 100644 --- a/Eigen/src/Geometry/AlignedBox.h +++ b/Eigen/src/Geometry/AlignedBox.h @@ -71,7 +71,11 @@ EIGEN_MAKE_ALIGNED_OPERATOR_NEW_IF_VECTORIZABLE_FIXED_SIZE(_Scalar,_AmbientDim) template<typename Derived> inline explicit AlignedBox(const MatrixBase<Derived>& a_p) { +#ifndef EIGEN_TEST_EVALUATORS typename internal::nested<Derived,2>::type p(a_p.derived()); +#else + typename internal::nested_eval<Derived,2>::type p(a_p.derived()); +#endif m_min = p; m_max = p; } diff --git a/Eigen/src/Geometry/Homogeneous.h b/Eigen/src/Geometry/Homogeneous.h index 00e71d190..7a3b5de80 100644 --- a/Eigen/src/Geometry/Homogeneous.h +++ b/Eigen/src/Geometry/Homogeneous.h @@ -48,8 +48,10 @@ struct traits<Homogeneous<MatrixType,Direction> > TmpFlags = _MatrixTypeNested::Flags & HereditaryBits, Flags = ColsAtCompileTime==1 ? (TmpFlags & ~RowMajorBit) : RowsAtCompileTime==1 ? (TmpFlags | RowMajorBit) - : TmpFlags, - CoeffReadCost = _MatrixTypeNested::CoeffReadCost + : TmpFlags +#ifndef EIGEN_TEST_EVALUATORS + , CoeffReadCost = _MatrixTypeNested::CoeffReadCost +#endif // EIGEN_TEST_EVALUATORS }; }; @@ -63,6 +65,7 @@ template<typename MatrixType,int _Direction> class Homogeneous { public: + typedef MatrixType NestedExpression; enum { Direction = _Direction }; typedef MatrixBase<Homogeneous> Base; @@ -74,7 +77,10 @@ template<typename MatrixType,int _Direction> class Homogeneous inline Index rows() const { return m_matrix.rows() + (int(Direction)==Vertical ? 1 : 0); } inline Index cols() const { return m_matrix.cols() + (int(Direction)==Horizontal ? 1 : 0); } + + const NestedExpression& nestedExpression() const { return m_matrix; } +#ifndef EIGEN_TEST_EVALUATORS inline Scalar coeff(Index row, Index col) const { if( (int(Direction)==Vertical && row==m_matrix.rows()) @@ -106,6 +112,31 @@ template<typename MatrixType,int _Direction> class Homogeneous eigen_assert(int(Direction)==Vertical); return internal::homogeneous_left_product_impl<Homogeneous,Transform<Scalar,Dim,Mode,Options> >(lhs,rhs.m_matrix); } +#else + template<typename Rhs> + inline const Product<Homogeneous,Rhs> + operator* (const MatrixBase<Rhs>& rhs) const + { + eigen_assert(int(Direction)==Horizontal); + return Product<Homogeneous,Rhs>(*this,rhs.derived()); + } + + template<typename Lhs> friend + inline const Product<Lhs,Homogeneous> + operator* (const MatrixBase<Lhs>& lhs, const Homogeneous& rhs) + { + eigen_assert(int(Direction)==Vertical); + return Product<Lhs,Homogeneous>(lhs.derived(),rhs); + } + + template<typename Scalar, int Dim, int Mode, int Options> friend + inline const Product<Transform<Scalar,Dim,Mode,Options>, Homogeneous > + operator* (const Transform<Scalar,Dim,Mode,Options>& lhs, const Homogeneous& rhs) + { + eigen_assert(int(Direction)==Vertical); + return Product<Transform<Scalar,Dim,Mode,Options>, Homogeneous>(lhs,rhs); + } +#endif protected: typename MatrixType::Nested m_matrix; @@ -300,6 +331,102 @@ struct homogeneous_right_product_impl<Homogeneous<MatrixType,Horizontal>,Rhs> typename Rhs::Nested m_rhs; }; +#ifdef EIGEN_TEST_EVALUATORS + +template<typename ArgType,int Direction> +struct evaluator_traits<Homogeneous<ArgType,Direction> > +{ + typedef typename storage_kind_to_evaluator_kind<typename ArgType::StorageKind>::Kind Kind; + typedef HomogeneousShape Shape; + static const int AssumeAliasing = 0; +}; + +template<> struct AssignmentKind<DenseShape,HomogeneousShape> { typedef Dense2Dense Kind; }; + + +template<typename ArgType,int Direction> +struct unary_evaluator<Homogeneous<ArgType,Direction>, IndexBased> + : evaluator<typename Homogeneous<ArgType,Direction>::PlainObject >::type +{ + typedef Homogeneous<ArgType,Direction> XprType; + typedef typename XprType::PlainObject PlainObject; + typedef typename evaluator<PlainObject>::type Base; + + unary_evaluator(const XprType& op) + : Base(), m_temp(op) + { + ::new (static_cast<Base*>(this)) Base(m_temp); + } + +protected: + PlainObject m_temp; +}; + +// dense = homogeneous +template< typename DstXprType, typename ArgType, typename Scalar> +struct Assignment<DstXprType, Homogeneous<ArgType,Vertical>, internal::assign_op<Scalar>, Dense2Dense, Scalar> +{ + typedef Homogeneous<ArgType,Vertical> SrcXprType; + // TODO clang generates garbage if this function is inlined. no valgrind error though. +#ifdef __clang__ + EIGEN_DONT_INLINE +#endif + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + { + dst.template topRows<ArgType::RowsAtCompileTime>(src.nestedExpression().rows()) = src.nestedExpression(); + dst.row(dst.rows()-1).setOnes(); + } +}; + +// dense = homogeneous +template< typename DstXprType, typename ArgType, typename Scalar> +struct Assignment<DstXprType, Homogeneous<ArgType,Horizontal>, internal::assign_op<Scalar>, Dense2Dense, Scalar> +{ + typedef Homogeneous<ArgType,Horizontal> SrcXprType; + // TODO clang generates garbage if this function is inlined. no valgrind error though. +#ifdef __clang__ + EIGEN_DONT_INLINE +#endif + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + { + dst.template leftCols<ArgType::ColsAtCompileTime>(src.nestedExpression().cols()) = src.nestedExpression(); + dst.col(dst.cols()-1).setOnes(); + } +}; + +template<typename LhsArg, typename Rhs, int ProductTag> +struct generic_product_impl<Homogeneous<LhsArg,Horizontal>, Rhs, HomogeneousShape, DenseShape, ProductTag> +{ + template<typename Dest> + static void evalTo(Dest& dst, const Homogeneous<LhsArg,Horizontal>& lhs, const Rhs& rhs) + { + homogeneous_right_product_impl<Homogeneous<LhsArg,Horizontal>, Rhs>(lhs.nestedExpression(), rhs).evalTo(dst); + } +}; + +template<typename Lhs, typename RhsArg, int ProductTag> +struct generic_product_impl<Lhs, Homogeneous<RhsArg,Vertical>, DenseShape, HomogeneousShape, ProductTag> +{ + template<typename Dest> + static void evalTo(Dest& dst, const Lhs& lhs, const Homogeneous<RhsArg,Vertical>& rhs) + { + homogeneous_left_product_impl<Homogeneous<RhsArg,Vertical>, Lhs>(lhs, rhs.nestedExpression()).evalTo(dst); + } +}; + +template<typename Scalar, int Dim, int Mode,int Options, typename RhsArg, int ProductTag> +struct generic_product_impl<Transform<Scalar,Dim,Mode,Options>, Homogeneous<RhsArg,Vertical>, DenseShape, HomogeneousShape, ProductTag> +{ + typedef Transform<Scalar,Dim,Mode,Options> TransformType; + template<typename Dest> + static void evalTo(Dest& dst, const TransformType& lhs, const Homogeneous<RhsArg,Vertical>& rhs) + { + homogeneous_left_product_impl<Homogeneous<RhsArg,Vertical>, TransformType>(lhs, rhs.nestedExpression()).evalTo(dst); + } +}; + +#endif // EIGEN_TEST_EVALUATORS + } // end namespace internal } // end namespace Eigen diff --git a/Eigen/src/Geometry/OrthoMethods.h b/Eigen/src/Geometry/OrthoMethods.h index 26be3ee5b..98e7a9bfc 100644 --- a/Eigen/src/Geometry/OrthoMethods.h +++ b/Eigen/src/Geometry/OrthoMethods.h @@ -30,8 +30,13 @@ MatrixBase<Derived>::cross(const MatrixBase<OtherDerived>& other) const // Note that there is no need for an expression here since the compiler // optimize such a small temporary very well (even within a complex expression) +#ifndef EIGEN_TEST_EVALUATORS typename internal::nested<Derived,2>::type lhs(derived()); typename internal::nested<OtherDerived,2>::type rhs(other.derived()); +#else + typename internal::nested_eval<Derived,2>::type lhs(derived()); + typename internal::nested_eval<OtherDerived,2>::type rhs(other.derived()); +#endif return typename cross_product_return_type<OtherDerived>::type( numext::conj(lhs.coeff(1) * rhs.coeff(2) - lhs.coeff(2) * rhs.coeff(1)), numext::conj(lhs.coeff(2) * rhs.coeff(0) - lhs.coeff(0) * rhs.coeff(2)), @@ -76,8 +81,13 @@ MatrixBase<Derived>::cross3(const MatrixBase<OtherDerived>& other) const EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(Derived,4) EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(OtherDerived,4) +#ifndef EIGEN_TEST_EVALUATORS typedef typename internal::nested<Derived,2>::type DerivedNested; typedef typename internal::nested<OtherDerived,2>::type OtherDerivedNested; +#else + typedef typename internal::nested_eval<Derived,2>::type DerivedNested; + typedef typename internal::nested_eval<OtherDerived,2>::type OtherDerivedNested; +#endif DerivedNested lhs(derived()); OtherDerivedNested rhs(other.derived()); @@ -103,21 +113,29 @@ VectorwiseOp<ExpressionType,Direction>::cross(const MatrixBase<OtherDerived>& ot EIGEN_STATIC_ASSERT_VECTOR_SPECIFIC_SIZE(OtherDerived,3) EIGEN_STATIC_ASSERT((internal::is_same<Scalar, typename OtherDerived::Scalar>::value), YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) + +#ifndef EIGEN_TEST_EVALUATORS + typename internal::nested<ExpressionType,2>::type mat(_expression()); + typename internal::nested<OtherDerived,2>::type vec(other.derived()); +#else + typename internal::nested_eval<ExpressionType,2>::type mat(_expression()); + typename internal::nested_eval<OtherDerived,2>::type vec(other.derived()); +#endif CrossReturnType res(_expression().rows(),_expression().cols()); if(Direction==Vertical) { eigen_assert(CrossReturnType::RowsAtCompileTime==3 && "the matrix must have exactly 3 rows"); - res.row(0) = (_expression().row(1) * other.coeff(2) - _expression().row(2) * other.coeff(1)).conjugate(); - res.row(1) = (_expression().row(2) * other.coeff(0) - _expression().row(0) * other.coeff(2)).conjugate(); - res.row(2) = (_expression().row(0) * other.coeff(1) - _expression().row(1) * other.coeff(0)).conjugate(); + res.row(0) = (mat.row(1) * vec.coeff(2) - mat.row(2) * vec.coeff(1)).conjugate(); + res.row(1) = (mat.row(2) * vec.coeff(0) - mat.row(0) * vec.coeff(2)).conjugate(); + res.row(2) = (mat.row(0) * vec.coeff(1) - mat.row(1) * vec.coeff(0)).conjugate(); } else { eigen_assert(CrossReturnType::ColsAtCompileTime==3 && "the matrix must have exactly 3 columns"); - res.col(0) = (_expression().col(1) * other.coeff(2) - _expression().col(2) * other.coeff(1)).conjugate(); - res.col(1) = (_expression().col(2) * other.coeff(0) - _expression().col(0) * other.coeff(2)).conjugate(); - res.col(2) = (_expression().col(0) * other.coeff(1) - _expression().col(1) * other.coeff(0)).conjugate(); + res.col(0) = (mat.col(1) * vec.coeff(2) - mat.col(2) * vec.coeff(1)).conjugate(); + res.col(1) = (mat.col(2) * vec.coeff(0) - mat.col(0) * vec.coeff(2)).conjugate(); + res.col(2) = (mat.col(0) * vec.coeff(1) - mat.col(1) * vec.coeff(0)).conjugate(); } return res; } diff --git a/Eigen/src/Geometry/Transform.h b/Eigen/src/Geometry/Transform.h index cb93acf6b..bcf3e2723 100644 --- a/Eigen/src/Geometry/Transform.h +++ b/Eigen/src/Geometry/Transform.h @@ -62,6 +62,24 @@ struct transform_construct_from_matrix; template<typename TransformType> struct transform_take_affine_part; +#ifdef EIGEN_TEST_EVALUATORS +template<typename _Scalar, int _Dim, int _Mode, int _Options> +struct traits<Transform<_Scalar,_Dim,_Mode,_Options> > +{ + typedef _Scalar Scalar; + typedef DenseIndex Index; + typedef Dense StorageKind; + enum { + Dim1 = _Dim==Dynamic ? _Dim : _Dim + 1, + RowsAtCompileTime = _Mode==Projective ? Dim1 : _Dim, + ColsAtCompileTime = Dim1, + MaxRowsAtCompileTime = RowsAtCompileTime, + MaxColsAtCompileTime = ColsAtCompileTime, + Flags = 0 + }; +}; +#endif + } // end namespace internal /** \geometry_module \ingroup Geometry_Module @@ -355,6 +373,11 @@ public: inline Transform& operator=(const QTransform& other); inline QTransform toQTransform(void) const; #endif + +#ifdef EIGEN_TEST_EVALUATORS + Index rows() const { return int(Mode)==int(Projective) ? m_matrix.cols() : (m_matrix.cols()-1); } + Index cols() const { return m_matrix.cols(); } +#endif /** shortcut for m_matrix(row,col); * \sa MatrixBase::operator(Index,Index) const */ diff --git a/Eigen/src/Householder/HouseholderSequence.h b/Eigen/src/Householder/HouseholderSequence.h index d800ca1fa..99d3eb21f 100644 --- a/Eigen/src/Householder/HouseholderSequence.h +++ b/Eigen/src/Householder/HouseholderSequence.h @@ -73,6 +73,18 @@ struct traits<HouseholderSequence<VectorsType,CoeffsType,Side> > }; }; +#ifdef EIGEN_TEST_EVALUATORS + +struct HouseholderSequenceShape {}; + +template<typename VectorsType, typename CoeffsType, int Side> +struct evaluator_traits<HouseholderSequence<VectorsType,CoeffsType,Side> > + : public evaluator_traits_base<HouseholderSequence<VectorsType,CoeffsType,Side> > +{ + typedef HouseholderSequenceShape Shape; +}; +#endif + template<typename VectorsType, typename CoeffsType, int Side> struct hseq_side_dependent_impl { diff --git a/Eigen/src/LU/Determinant.h b/Eigen/src/LU/Determinant.h index bb8e78a8a..9726bd96a 100644 --- a/Eigen/src/LU/Determinant.h +++ b/Eigen/src/LU/Determinant.h @@ -92,7 +92,11 @@ template<typename Derived> inline typename internal::traits<Derived>::Scalar MatrixBase<Derived>::determinant() const { eigen_assert(rows() == cols()); +#ifdef EIGEN_TEST_EVALUATORS + typedef typename internal::nested_eval<Derived,Base::RowsAtCompileTime>::type Nested; +#else typedef typename internal::nested<Derived,Base::RowsAtCompileTime>::type Nested; +#endif return internal::determinant_impl<typename internal::remove_all<Nested>::type>::run(derived()); } diff --git a/Eigen/src/LU/FullPivLU.h b/Eigen/src/LU/FullPivLU.h index 971b9da1d..daf99e305 100644 --- a/Eigen/src/LU/FullPivLU.h +++ b/Eigen/src/LU/FullPivLU.h @@ -12,6 +12,15 @@ namespace Eigen { +namespace internal { +template<typename _MatrixType> struct traits<FullPivLU<_MatrixType> > + : traits<_MatrixType> +{ + enum { Flags = 0 }; +}; + +} // end namespace internal + /** \ingroup LU_Module * * \class FullPivLU @@ -62,6 +71,7 @@ template<typename _MatrixType> class FullPivLU typedef typename internal::plain_col_type<MatrixType, Index>::type IntColVectorType; typedef PermutationMatrix<ColsAtCompileTime, MaxColsAtCompileTime> PermutationQType; typedef PermutationMatrix<RowsAtCompileTime, MaxRowsAtCompileTime> PermutationPType; + typedef typename MatrixType::PlainObject PlainObject; /** * \brief Default Constructor. @@ -210,6 +220,15 @@ template<typename _MatrixType> class FullPivLU * * \sa TriangularView::solve(), kernel(), inverse() */ +#ifdef EIGEN_TEST_EVALUATORS + template<typename Rhs> + inline const Solve<FullPivLU, Rhs> + solve(const MatrixBase<Rhs>& b) const + { + eigen_assert(m_isInitialized && "LU is not initialized."); + return Solve<FullPivLU, Rhs>(*this, b.derived()); + } +#else template<typename Rhs> inline const internal::solve_retval<FullPivLU, Rhs> solve(const MatrixBase<Rhs>& b) const @@ -217,6 +236,7 @@ template<typename _MatrixType> class FullPivLU eigen_assert(m_isInitialized && "LU is not initialized."); return internal::solve_retval<FullPivLU, Rhs>(*this, b.derived()); } +#endif /** \returns the determinant of the matrix of which * *this is the LU decomposition. It has only linear complexity @@ -360,6 +380,14 @@ template<typename _MatrixType> class FullPivLU * * \sa MatrixBase::inverse() */ +#ifdef EIGEN_TEST_EVALUATORS + inline const Inverse<FullPivLU> inverse() const + { + eigen_assert(m_isInitialized && "LU is not initialized."); + eigen_assert(m_lu.rows() == m_lu.cols() && "You can't take the inverse of a non-square matrix!"); + return Inverse<FullPivLU>(*this); + } +#else inline const internal::solve_retval<FullPivLU,typename MatrixType::IdentityReturnType> inverse() const { eigen_assert(m_isInitialized && "LU is not initialized."); @@ -367,11 +395,18 @@ template<typename _MatrixType> class FullPivLU return internal::solve_retval<FullPivLU,typename MatrixType::IdentityReturnType> (*this, MatrixType::Identity(m_lu.rows(), m_lu.cols())); } +#endif MatrixType reconstructedMatrix() const; inline Index rows() const { return m_lu.rows(); } inline Index cols() const { return m_lu.cols(); } + + #ifndef EIGEN_PARSED_BY_DOXYGEN + template<typename RhsType, typename DstType> + EIGEN_DEVICE_FUNC + void _solve_impl(const RhsType &rhs, DstType &dst) const; + #endif protected: MatrixType m_lu; @@ -663,6 +698,61 @@ struct image_retval<FullPivLU<_MatrixType> > /***** Implementation of solve() *****************************************************/ +} // end namespace internal + +#ifndef EIGEN_PARSED_BY_DOXYGEN +template<typename _MatrixType> +template<typename RhsType, typename DstType> +void FullPivLU<_MatrixType>::_solve_impl(const RhsType &rhs, DstType &dst) const +{ + /* The decomposition PAQ = LU can be rewritten as A = P^{-1} L U Q^{-1}. + * So we proceed as follows: + * Step 1: compute c = P * rhs. + * Step 2: replace c by the solution x to Lx = c. Exists because L is invertible. + * Step 3: replace c by the solution x to Ux = c. May or may not exist. + * Step 4: result = Q * c; + */ + + const Index rows = this->rows(), + cols = this->cols(), + nonzero_pivots = this->nonzeroPivots(); + eigen_assert(rhs.rows() == rows); + const Index smalldim = (std::min)(rows, cols); + + if(nonzero_pivots == 0) + { + dst.setZero(); + return; + } + + typename RhsType::PlainObject c(rhs.rows(), rhs.cols()); + + // Step 1 + c = permutationP() * rhs; + + // Step 2 + m_lu.topLeftCorner(smalldim,smalldim) + .template triangularView<UnitLower>() + .solveInPlace(c.topRows(smalldim)); + if(rows>cols) + c.bottomRows(rows-cols) -= m_lu.bottomRows(rows-cols) * c.topRows(cols); + + // Step 3 + m_lu.topLeftCorner(nonzero_pivots, nonzero_pivots) + .template triangularView<Upper>() + .solveInPlace(c.topRows(nonzero_pivots)); + + // Step 4 + for(Index i = 0; i < nonzero_pivots; ++i) + dst.row(permutationQ().indices().coeff(i)) = c.row(i); + for(Index i = nonzero_pivots; i < m_lu.cols(); ++i) + dst.row(permutationQ().indices().coeff(i)).setZero(); +} +#endif + +namespace internal { + +#ifndef EIGEN_TEST_EVALUATORS template<typename _MatrixType, typename Rhs> struct solve_retval<FullPivLU<_MatrixType>, Rhs> : solve_retval_base<FullPivLU<_MatrixType>, Rhs> @@ -671,56 +761,24 @@ struct solve_retval<FullPivLU<_MatrixType>, Rhs> template<typename Dest> void evalTo(Dest& dst) const { - /* The decomposition PAQ = LU can be rewritten as A = P^{-1} L U Q^{-1}. - * So we proceed as follows: - * Step 1: compute c = P * rhs. - * Step 2: replace c by the solution x to Lx = c. Exists because L is invertible. - * Step 3: replace c by the solution x to Ux = c. May or may not exist. - * Step 4: result = Q * c; - */ - - const Index rows = dec().rows(), cols = dec().cols(), - nonzero_pivots = dec().nonzeroPivots(); - eigen_assert(rhs().rows() == rows); - const Index smalldim = (std::min)(rows, cols); - - if(nonzero_pivots == 0) - { - dst.setZero(); - return; - } - - typename Rhs::PlainObject c(rhs().rows(), rhs().cols()); - - // Step 1 - c = dec().permutationP() * rhs(); - - // Step 2 - dec().matrixLU() - .topLeftCorner(smalldim,smalldim) - .template triangularView<UnitLower>() - .solveInPlace(c.topRows(smalldim)); - if(rows>cols) - { - c.bottomRows(rows-cols) - -= dec().matrixLU().bottomRows(rows-cols) - * c.topRows(cols); - } - - // Step 3 - dec().matrixLU() - .topLeftCorner(nonzero_pivots, nonzero_pivots) - .template triangularView<Upper>() - .solveInPlace(c.topRows(nonzero_pivots)); - - // Step 4 - for(Index i = 0; i < nonzero_pivots; ++i) - dst.row(dec().permutationQ().indices().coeff(i)) = c.row(i); - for(Index i = nonzero_pivots; i < dec().matrixLU().cols(); ++i) - dst.row(dec().permutationQ().indices().coeff(i)).setZero(); + dec()._solve_impl(rhs(), dst); } }; +#endif +/***** Implementation of inverse() *****************************************************/ +#ifdef EIGEN_TEST_EVALUATORS +template<typename DstXprType, typename MatrixType, typename Scalar> +struct Assignment<DstXprType, Inverse<FullPivLU<MatrixType> >, internal::assign_op<Scalar>, Dense2Dense, Scalar> +{ + typedef FullPivLU<MatrixType> LuType; + typedef Inverse<LuType> SrcXprType; + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + { + dst = src.nestedExpression().solve(MatrixType::Identity(src.rows(), src.cols())); + } +}; +#endif } // end namespace internal /******* MatrixBase methods *****************************************************************/ diff --git a/Eigen/src/LU/Inverse.h b/Eigen/src/LU/InverseImpl.h index 8d1364e0a..e10fee48f 100644 --- a/Eigen/src/LU/Inverse.h +++ b/Eigen/src/LU/InverseImpl.h @@ -2,13 +2,14 @@ // for linear algebra. // // Copyright (C) 2008-2010 Benoit Jacob <jacob.benoit.1@gmail.com> +// Copyright (C) 2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -#ifndef EIGEN_INVERSE_H -#define EIGEN_INVERSE_H +#ifndef EIGEN_INVERSE_IMPL_H +#define EIGEN_INVERSE_IMPL_H namespace Eigen { @@ -42,7 +43,12 @@ struct compute_inverse<MatrixType, ResultType, 1> static inline void run(const MatrixType& matrix, ResultType& result) { typedef typename MatrixType::Scalar Scalar; +#ifdef EIGEN_TEST_EVALUATORS + typename internal::evaluator<MatrixType>::type matrixEval(matrix); + result.coeffRef(0,0) = Scalar(1) / matrixEval.coeff(0,0); +#else result.coeffRef(0,0) = Scalar(1) / matrix.coeff(0,0); +#endif } }; @@ -75,10 +81,10 @@ inline void compute_inverse_size2_helper( const MatrixType& matrix, const typename ResultType::Scalar& invdet, ResultType& result) { - result.coeffRef(0,0) = matrix.coeff(1,1) * invdet; + result.coeffRef(0,0) = matrix.coeff(1,1) * invdet; result.coeffRef(1,0) = -matrix.coeff(1,0) * invdet; result.coeffRef(0,1) = -matrix.coeff(0,1) * invdet; - result.coeffRef(1,1) = matrix.coeff(0,0) * invdet; + result.coeffRef(1,1) = matrix.coeff(0,0) * invdet; } template<typename MatrixType, typename ResultType> @@ -279,6 +285,7 @@ struct compute_inverse_and_det_with_check<MatrixType, ResultType, 4> *** MatrixBase methods *** *************************/ +#ifndef EIGEN_TEST_EVALUATORS template<typename MatrixType> struct traits<inverse_impl<MatrixType> > { @@ -313,9 +320,40 @@ struct inverse_impl : public ReturnByValue<inverse_impl<MatrixType> > compute_inverse<MatrixTypeNestedCleaned, Dest>::run(m_matrix, dst); } }; +#endif +} // end namespace internal + +#ifdef EIGEN_TEST_EVALUATORS + +namespace internal { + +// Specialization for "dense = dense_xpr.inverse()" +template<typename DstXprType, typename XprType, typename Scalar> +struct Assignment<DstXprType, Inverse<XprType>, internal::assign_op<Scalar>, Dense2Dense, Scalar> +{ + typedef Inverse<XprType> SrcXprType; + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + { + // FIXME shall we resize dst here? + const int Size = EIGEN_PLAIN_ENUM_MIN(XprType::ColsAtCompileTime,DstXprType::ColsAtCompileTime); + EIGEN_ONLY_USED_FOR_DEBUG(Size); + eigen_assert(( (Size<=1) || (Size>4) || (extract_data(src.nestedExpression())!=extract_data(dst))) + && "Aliasing problem detected in inverse(), you need to do inverse().eval() here."); + typedef typename internal::nested_eval<XprType,XprType::ColsAtCompileTime>::type ActualXprType; + typedef typename internal::remove_all<ActualXprType>::type ActualXprTypeCleanded; + + ActualXprType actual_xpr(src.nestedExpression()); + + compute_inverse<ActualXprTypeCleanded, DstXprType>::run(actual_xpr, dst); + } +}; + + } // end namespace internal +#endif + /** \lu_module * * \returns the matrix inverse of this matrix. @@ -333,6 +371,15 @@ struct inverse_impl : public ReturnByValue<inverse_impl<MatrixType> > * * \sa computeInverseAndDetWithCheck() */ +#ifdef EIGEN_TEST_EVALUATORS +template<typename Derived> +inline const Inverse<Derived> MatrixBase<Derived>::inverse() const +{ + EIGEN_STATIC_ASSERT(!NumTraits<Scalar>::IsInteger,THIS_FUNCTION_IS_NOT_FOR_INTEGER_NUMERIC_TYPES) + eigen_assert(rows() == cols()); + return Inverse<Derived>(derived()); +} +#else template<typename Derived> inline const internal::inverse_impl<Derived> MatrixBase<Derived>::inverse() const { @@ -340,6 +387,7 @@ inline const internal::inverse_impl<Derived> MatrixBase<Derived>::inverse() cons eigen_assert(rows() == cols()); return internal::inverse_impl<Derived>(derived()); } +#endif /** \lu_module * @@ -374,7 +422,11 @@ inline void MatrixBase<Derived>::computeInverseAndDetWithCheck( // for larger sizes, evaluating has negligible cost and limits code size. typedef typename internal::conditional< RowsAtCompileTime == 2, +#ifndef EIGEN_TEST_EVALUATORS typename internal::remove_all<typename internal::nested<Derived, 2>::type>::type, +#else + typename internal::remove_all<typename internal::nested_eval<Derived, 2>::type>::type, +#endif PlainObject >::type MatrixType; internal::compute_inverse_and_det_with_check<MatrixType, ResultType>::run @@ -414,4 +466,4 @@ inline void MatrixBase<Derived>::computeInverseWithCheck( } // end namespace Eigen -#endif // EIGEN_INVERSE_H +#endif // EIGEN_INVERSE_IMPL_H diff --git a/Eigen/src/LU/PartialPivLU.h b/Eigen/src/LU/PartialPivLU.h index 2f65c3a49..57076f3a4 100644 --- a/Eigen/src/LU/PartialPivLU.h +++ b/Eigen/src/LU/PartialPivLU.h @@ -13,6 +13,19 @@ namespace Eigen { +namespace internal { +template<typename _MatrixType> struct traits<PartialPivLU<_MatrixType> > + : traits<_MatrixType> +{ + typedef traits<_MatrixType> BaseTraits; + enum { + Flags = BaseTraits::Flags & RowMajorBit, + CoeffReadCost = Dynamic + }; +}; + +} // end namespace internal + /** \ingroup LU_Module * * \class PartialPivLU @@ -62,6 +75,7 @@ template<typename _MatrixType> class PartialPivLU typedef typename MatrixType::Index Index; typedef PermutationMatrix<RowsAtCompileTime, MaxRowsAtCompileTime> PermutationType; typedef Transpositions<RowsAtCompileTime, MaxRowsAtCompileTime> TranspositionType; + typedef typename MatrixType::PlainObject PlainObject; /** @@ -128,6 +142,15 @@ template<typename _MatrixType> class PartialPivLU * * \sa TriangularView::solve(), inverse(), computeInverse() */ +#ifdef EIGEN_TEST_EVALUATORS + template<typename Rhs> + inline const Solve<PartialPivLU, Rhs> + solve(const MatrixBase<Rhs>& b) const + { + eigen_assert(m_isInitialized && "PartialPivLU is not initialized."); + return Solve<PartialPivLU, Rhs>(*this, b.derived()); + } +#else template<typename Rhs> inline const internal::solve_retval<PartialPivLU, Rhs> solve(const MatrixBase<Rhs>& b) const @@ -135,6 +158,7 @@ template<typename _MatrixType> class PartialPivLU eigen_assert(m_isInitialized && "PartialPivLU is not initialized."); return internal::solve_retval<PartialPivLU, Rhs>(*this, b.derived()); } +#endif /** \returns the inverse of the matrix of which *this is the LU decomposition. * @@ -143,12 +167,20 @@ template<typename _MatrixType> class PartialPivLU * * \sa MatrixBase::inverse(), LU::inverse() */ +#ifdef EIGEN_TEST_EVALUATORS + inline const Inverse<PartialPivLU> inverse() const + { + eigen_assert(m_isInitialized && "PartialPivLU is not initialized."); + return Inverse<PartialPivLU>(*this); + } +#else inline const internal::solve_retval<PartialPivLU,typename MatrixType::IdentityReturnType> inverse() const { eigen_assert(m_isInitialized && "PartialPivLU is not initialized."); return internal::solve_retval<PartialPivLU,typename MatrixType::IdentityReturnType> (*this, MatrixType::Identity(m_lu.rows(), m_lu.cols())); } +#endif /** \returns the determinant of the matrix of which * *this is the LU decomposition. It has only linear complexity @@ -169,6 +201,30 @@ template<typename _MatrixType> class PartialPivLU inline Index rows() const { return m_lu.rows(); } inline Index cols() const { return m_lu.cols(); } + + #ifndef EIGEN_PARSED_BY_DOXYGEN + template<typename RhsType, typename DstType> + EIGEN_DEVICE_FUNC + void _solve_impl(const RhsType &rhs, DstType &dst) const { + /* The decomposition PA = LU can be rewritten as A = P^{-1} L U. + * So we proceed as follows: + * Step 1: compute c = Pb. + * Step 2: replace c by the solution x to Lx = c. + * Step 3: replace c by the solution x to Ux = c. + */ + + eigen_assert(rhs.rows() == m_lu.rows()); + + // Step 1 + dst = permutationP() * rhs; + + // Step 2 + m_lu.template triangularView<UnitLower>().solveInPlace(dst); + + // Step 3 + m_lu.template triangularView<Upper>().solveInPlace(dst); + } + #endif protected: MatrixType m_lu; @@ -434,6 +490,7 @@ MatrixType PartialPivLU<MatrixType>::reconstructedMatrix() const namespace internal { +#ifndef EIGEN_TEST_EVALUATORS template<typename _MatrixType, typename Rhs> struct solve_retval<PartialPivLU<_MatrixType>, Rhs> : solve_retval_base<PartialPivLU<_MatrixType>, Rhs> @@ -442,26 +499,24 @@ struct solve_retval<PartialPivLU<_MatrixType>, Rhs> template<typename Dest> void evalTo(Dest& dst) const { - /* The decomposition PA = LU can be rewritten as A = P^{-1} L U. - * So we proceed as follows: - * Step 1: compute c = Pb. - * Step 2: replace c by the solution x to Lx = c. - * Step 3: replace c by the solution x to Ux = c. - */ - - eigen_assert(rhs().rows() == dec().matrixLU().rows()); - - // Step 1 - dst = dec().permutationP() * rhs(); - - // Step 2 - dec().matrixLU().template triangularView<UnitLower>().solveInPlace(dst); - - // Step 3 - dec().matrixLU().template triangularView<Upper>().solveInPlace(dst); + dec()._solve_impl(rhs(), dst); } }; +#endif +/***** Implementation of inverse() *****************************************************/ +#ifdef EIGEN_TEST_EVALUATORS +template<typename DstXprType, typename MatrixType, typename Scalar> +struct Assignment<DstXprType, Inverse<PartialPivLU<MatrixType> >, internal::assign_op<Scalar>, Dense2Dense, Scalar> +{ + typedef PartialPivLU<MatrixType> LuType; + typedef Inverse<LuType> SrcXprType; + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + { + dst = src.nestedExpression().solve(MatrixType::Identity(src.rows(), src.cols())); + } +}; +#endif } // end namespace internal /******** MatrixBase methods *******/ diff --git a/Eigen/src/LU/arch/Inverse_SSE.h b/Eigen/src/LU/arch/Inverse_SSE.h index 60b7a2376..1f62ef14e 100644 --- a/Eigen/src/LU/arch/Inverse_SSE.h +++ b/Eigen/src/LU/arch/Inverse_SSE.h @@ -39,9 +39,11 @@ struct compute_inverse_size4<Architecture::SSE, float, MatrixType, ResultType> ResultAlignment = bool(ResultType::Flags&AlignedBit), StorageOrdersMatch = (MatrixType::Flags&RowMajorBit) == (ResultType::Flags&RowMajorBit) }; + typedef typename conditional<(MatrixType::Flags&LinearAccessBit),MatrixType const &,typename MatrixType::PlainObject>::type ActualMatrixType; - static void run(const MatrixType& matrix, ResultType& result) + static void run(const MatrixType& mat, ResultType& result) { + ActualMatrixType matrix(mat); EIGEN_ALIGN16 const unsigned int _Sign_PNNP[4] = { 0x00000000, 0x80000000, 0x80000000, 0x00000000 }; // Load the full matrix into registers @@ -167,14 +169,17 @@ struct compute_inverse_size4<Architecture::SSE, double, MatrixType, ResultType> ResultAlignment = bool(ResultType::Flags&AlignedBit), StorageOrdersMatch = (MatrixType::Flags&RowMajorBit) == (ResultType::Flags&RowMajorBit) }; - static void run(const MatrixType& matrix, ResultType& result) + typedef typename conditional<(MatrixType::Flags&LinearAccessBit),MatrixType const &,typename MatrixType::PlainObject>::type ActualMatrixType; + + static void run(const MatrixType& mat, ResultType& result) { + ActualMatrixType matrix(mat); const __m128d _Sign_NP = _mm_castsi128_pd(_mm_set_epi32(0x0,0x0,0x80000000,0x0)); const __m128d _Sign_PN = _mm_castsi128_pd(_mm_set_epi32(0x80000000,0x0,0x0,0x0)); // The inverse is calculated using "Divide and Conquer" technique. The // original matrix is divide into four 2x2 sub-matrices. Since each - // register of the matrix holds two element, the smaller matrices are + // register of the matrix holds two elements, the smaller matrices are // consisted of two registers. Hence we get a better locality of the // calculations. diff --git a/Eigen/src/QR/ColPivHouseholderQR.h b/Eigen/src/QR/ColPivHouseholderQR.h index 4824880f5..96904c65f 100644 --- a/Eigen/src/QR/ColPivHouseholderQR.h +++ b/Eigen/src/QR/ColPivHouseholderQR.h @@ -13,6 +13,15 @@ namespace Eigen { +namespace internal { +template<typename _MatrixType> struct traits<ColPivHouseholderQR<_MatrixType> > + : traits<_MatrixType> +{ + enum { Flags = 0 }; +}; + +} // end namespace internal + /** \ingroup QR_Module * * \class ColPivHouseholderQR @@ -56,6 +65,7 @@ template<typename _MatrixType> class ColPivHouseholderQR typedef typename internal::plain_row_type<MatrixType>::type RowVectorType; typedef typename internal::plain_row_type<MatrixType, RealScalar>::type RealRowVectorType; typedef HouseholderSequence<MatrixType,typename internal::remove_all<typename HCoeffsType::ConjugateReturnType>::type> HouseholderSequenceType; + typedef typename MatrixType::PlainObject PlainObject; private: @@ -137,6 +147,15 @@ template<typename _MatrixType> class ColPivHouseholderQR * Example: \include ColPivHouseholderQR_solve.cpp * Output: \verbinclude ColPivHouseholderQR_solve.out */ +#ifdef EIGEN_TEST_EVALUATORS + template<typename Rhs> + inline const Solve<ColPivHouseholderQR, Rhs> + solve(const MatrixBase<Rhs>& b) const + { + eigen_assert(m_isInitialized && "ColPivHouseholderQR is not initialized."); + return Solve<ColPivHouseholderQR, Rhs>(*this, b.derived()); + } +#else template<typename Rhs> inline const internal::solve_retval<ColPivHouseholderQR, Rhs> solve(const MatrixBase<Rhs>& b) const @@ -144,9 +163,10 @@ template<typename _MatrixType> class ColPivHouseholderQR eigen_assert(m_isInitialized && "ColPivHouseholderQR is not initialized."); return internal::solve_retval<ColPivHouseholderQR, Rhs>(*this, b.derived()); } +#endif - HouseholderSequenceType householderQ(void) const; - HouseholderSequenceType matrixQ(void) const + HouseholderSequenceType householderQ() const; + HouseholderSequenceType matrixQ() const { return householderQ(); } @@ -284,6 +304,13 @@ template<typename _MatrixType> class ColPivHouseholderQR * \note If this matrix is not invertible, the returned matrix has undefined coefficients. * Use isInvertible() to first determine whether this matrix is invertible. */ +#ifdef EIGEN_TEST_EVALUATORS + inline const Inverse<ColPivHouseholderQR> inverse() const + { + eigen_assert(m_isInitialized && "ColPivHouseholderQR is not initialized."); + return Inverse<ColPivHouseholderQR>(*this); + } +#else inline const internal::solve_retval<ColPivHouseholderQR, typename MatrixType::IdentityReturnType> inverse() const @@ -292,6 +319,7 @@ template<typename _MatrixType> class ColPivHouseholderQR return internal::solve_retval<ColPivHouseholderQR,typename MatrixType::IdentityReturnType> (*this, MatrixType::Identity(m_qr.rows(), m_qr.cols())); } +#endif inline Index rows() const { return m_qr.rows(); } inline Index cols() const { return m_qr.cols(); } @@ -382,6 +410,12 @@ template<typename _MatrixType> class ColPivHouseholderQR eigen_assert(m_isInitialized && "Decomposition is not initialized."); return Success; } + + #ifndef EIGEN_PARSED_BY_DOXYGEN + template<typename RhsType, typename DstType> + EIGEN_DEVICE_FUNC + void _solve_impl(const RhsType &rhs, DstType &dst) const; + #endif protected: MatrixType m_qr; @@ -514,8 +548,41 @@ ColPivHouseholderQR<MatrixType>& ColPivHouseholderQR<MatrixType>::compute(const return *this; } +#ifndef EIGEN_PARSED_BY_DOXYGEN +template<typename _MatrixType> +template<typename RhsType, typename DstType> +void ColPivHouseholderQR<_MatrixType>::_solve_impl(const RhsType &rhs, DstType &dst) const +{ + eigen_assert(rhs.rows() == rows()); + + const Index nonzero_pivots = nonzeroPivots(); + + if(nonzero_pivots == 0) + { + dst.setZero(); + return; + } + + typename RhsType::PlainObject c(rhs); + + // Note that the matrix Q = H_0^* H_1^*... so its inverse is Q^* = (H_0 H_1 ...)^T + c.applyOnTheLeft(householderSequence(m_qr, m_hCoeffs) + .setLength(nonzero_pivots) + .transpose() + ); + + m_qr.topLeftCorner(nonzero_pivots, nonzero_pivots) + .template triangularView<Upper>() + .solveInPlace(c.topRows(nonzero_pivots)); + + for(Index i = 0; i < nonzero_pivots; ++i) dst.row(m_colsPermutation.indices().coeff(i)) = c.row(i); + for(Index i = nonzero_pivots; i < cols(); ++i) dst.row(m_colsPermutation.indices().coeff(i)).setZero(); +} +#endif + namespace internal { +#ifndef EIGEN_TEST_EVALUATORS template<typename _MatrixType, typename Rhs> struct solve_retval<ColPivHouseholderQR<_MatrixType>, Rhs> : solve_retval_base<ColPivHouseholderQR<_MatrixType>, Rhs> @@ -524,34 +591,23 @@ struct solve_retval<ColPivHouseholderQR<_MatrixType>, Rhs> template<typename Dest> void evalTo(Dest& dst) const { - eigen_assert(rhs().rows() == dec().rows()); - - const Index cols = dec().cols(), - nonzero_pivots = dec().nonzeroPivots(); - - if(nonzero_pivots == 0) - { - dst.setZero(); - return; - } - - typename Rhs::PlainObject c(rhs()); - - // Note that the matrix Q = H_0^* H_1^*... so its inverse is Q^* = (H_0 H_1 ...)^T - c.applyOnTheLeft(householderSequence(dec().matrixQR(), dec().hCoeffs()) - .setLength(dec().nonzeroPivots()) - .transpose() - ); - - dec().matrixR() - .topLeftCorner(nonzero_pivots, nonzero_pivots) - .template triangularView<Upper>() - .solveInPlace(c.topRows(nonzero_pivots)); + dec()._solve_impl(rhs(), dst); + } +}; +#endif - for(Index i = 0; i < nonzero_pivots; ++i) dst.row(dec().colsPermutation().indices().coeff(i)) = c.row(i); - for(Index i = nonzero_pivots; i < cols; ++i) dst.row(dec().colsPermutation().indices().coeff(i)).setZero(); +#ifdef EIGEN_TEST_EVALUATORS +template<typename DstXprType, typename MatrixType, typename Scalar> +struct Assignment<DstXprType, Inverse<ColPivHouseholderQR<MatrixType> >, internal::assign_op<Scalar>, Dense2Dense, Scalar> +{ + typedef ColPivHouseholderQR<MatrixType> QrType; + typedef Inverse<QrType> SrcXprType; + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + { + dst = src.nestedExpression().solve(MatrixType::Identity(src.rows(), src.cols())); } }; +#endif } // end namespace internal diff --git a/Eigen/src/QR/FullPivHouseholderQR.h b/Eigen/src/QR/FullPivHouseholderQR.h index a7b0fc16f..ea15da041 100644 --- a/Eigen/src/QR/FullPivHouseholderQR.h +++ b/Eigen/src/QR/FullPivHouseholderQR.h @@ -15,6 +15,12 @@ namespace Eigen { namespace internal { +template<typename _MatrixType> struct traits<FullPivHouseholderQR<_MatrixType> > + : traits<_MatrixType> +{ + enum { Flags = 0 }; +}; + template<typename MatrixType> struct FullPivHouseholderQRMatrixQReturnType; template<typename MatrixType> @@ -23,7 +29,7 @@ struct traits<FullPivHouseholderQRMatrixQReturnType<MatrixType> > typedef typename MatrixType::PlainObject ReturnType; }; -} +} // end namespace internal /** \ingroup QR_Module * @@ -69,6 +75,7 @@ template<typename _MatrixType> class FullPivHouseholderQR typedef PermutationMatrix<ColsAtCompileTime, MaxColsAtCompileTime> PermutationType; typedef typename internal::plain_row_type<MatrixType>::type RowVectorType; typedef typename internal::plain_col_type<MatrixType>::type ColVectorType; + typedef typename MatrixType::PlainObject PlainObject; /** \brief Default Constructor. * @@ -144,6 +151,15 @@ template<typename _MatrixType> class FullPivHouseholderQR * Example: \include FullPivHouseholderQR_solve.cpp * Output: \verbinclude FullPivHouseholderQR_solve.out */ +#ifdef EIGEN_TEST_EVALUATORS + template<typename Rhs> + inline const Solve<FullPivHouseholderQR, Rhs> + solve(const MatrixBase<Rhs>& b) const + { + eigen_assert(m_isInitialized && "FullPivHouseholderQR is not initialized."); + return Solve<FullPivHouseholderQR, Rhs>(*this, b.derived()); + } +#else template<typename Rhs> inline const internal::solve_retval<FullPivHouseholderQR, Rhs> solve(const MatrixBase<Rhs>& b) const @@ -151,6 +167,7 @@ template<typename _MatrixType> class FullPivHouseholderQR eigen_assert(m_isInitialized && "FullPivHouseholderQR is not initialized."); return internal::solve_retval<FullPivHouseholderQR, Rhs>(*this, b.derived()); } +#endif /** \returns Expression object representing the matrix Q */ @@ -280,7 +297,15 @@ template<typename _MatrixType> class FullPivHouseholderQR * * \note If this matrix is not invertible, the returned matrix has undefined coefficients. * Use isInvertible() to first determine whether this matrix is invertible. - */ inline const + */ +#ifdef EIGEN_TEST_EVALUATORS + inline const Inverse<FullPivHouseholderQR> inverse() const + { + eigen_assert(m_isInitialized && "FullPivHouseholderQR is not initialized."); + return Inverse<FullPivHouseholderQR>(*this); + } +#else + inline const internal::solve_retval<FullPivHouseholderQR, typename MatrixType::IdentityReturnType> inverse() const { @@ -288,6 +313,7 @@ template<typename _MatrixType> class FullPivHouseholderQR return internal::solve_retval<FullPivHouseholderQR,typename MatrixType::IdentityReturnType> (*this, MatrixType::Identity(m_qr.rows(), m_qr.cols())); } +#endif inline Index rows() const { return m_qr.rows(); } inline Index cols() const { return m_qr.cols(); } @@ -366,6 +392,12 @@ template<typename _MatrixType> class FullPivHouseholderQR * diagonal coefficient of U. */ RealScalar maxPivot() const { return m_maxpivot; } + + #ifndef EIGEN_PARSED_BY_DOXYGEN + template<typename RhsType, typename DstType> + EIGEN_DEVICE_FUNC + void _solve_impl(const RhsType &rhs, DstType &dst) const; + #endif protected: MatrixType m_qr; @@ -485,8 +517,46 @@ FullPivHouseholderQR<MatrixType>& FullPivHouseholderQR<MatrixType>::compute(cons return *this; } -namespace internal { +#ifndef EIGEN_PARSED_BY_DOXYGEN +template<typename _MatrixType> +template<typename RhsType, typename DstType> +void FullPivHouseholderQR<_MatrixType>::_solve_impl(const RhsType &rhs, DstType &dst) const +{ + eigen_assert(rhs.rows() == rows()); + const Index l_rank = rank(); + + // FIXME introduce nonzeroPivots() and use it here. and more generally, + // make the same improvements in this dec as in FullPivLU. + if(l_rank==0) + { + dst.setZero(); + return; + } + typename RhsType::PlainObject c(rhs); + + Matrix<Scalar,1,RhsType::ColsAtCompileTime> temp(rhs.cols()); + for (Index k = 0; k < l_rank; ++k) + { + Index remainingSize = rows()-k; + c.row(k).swap(c.row(m_rows_transpositions.coeff(k))); + c.bottomRightCorner(remainingSize, rhs.cols()) + .applyHouseholderOnTheLeft(m_qr.col(k).tail(remainingSize-1), + m_hCoeffs.coeff(k), &temp.coeffRef(0)); + } + + m_qr.topLeftCorner(l_rank, l_rank) + .template triangularView<Upper>() + .solveInPlace(c.topRows(l_rank)); + + for(Index i = 0; i < l_rank; ++i) dst.row(m_cols_permutation.indices().coeff(i)) = c.row(i); + for(Index i = l_rank; i < cols(); ++i) dst.row(m_cols_permutation.indices().coeff(i)).setZero(); +} +#endif + +namespace internal { + +#ifndef EIGEN_TEST_EVALUATORS template<typename _MatrixType, typename Rhs> struct solve_retval<FullPivHouseholderQR<_MatrixType>, Rhs> : solve_retval_base<FullPivHouseholderQR<_MatrixType>, Rhs> @@ -495,38 +565,23 @@ struct solve_retval<FullPivHouseholderQR<_MatrixType>, Rhs> template<typename Dest> void evalTo(Dest& dst) const { - const Index rows = dec().rows(), cols = dec().cols(); - eigen_assert(rhs().rows() == rows); - - // FIXME introduce nonzeroPivots() and use it here. and more generally, - // make the same improvements in this dec as in FullPivLU. - if(dec().rank()==0) - { - dst.setZero(); - return; - } - - typename Rhs::PlainObject c(rhs()); - - Matrix<Scalar,1,Rhs::ColsAtCompileTime> temp(rhs().cols()); - for (Index k = 0; k < dec().rank(); ++k) - { - Index remainingSize = rows-k; - c.row(k).swap(c.row(dec().rowsTranspositions().coeff(k))); - c.bottomRightCorner(remainingSize, rhs().cols()) - .applyHouseholderOnTheLeft(dec().matrixQR().col(k).tail(remainingSize-1), - dec().hCoeffs().coeff(k), &temp.coeffRef(0)); - } - - dec().matrixQR() - .topLeftCorner(dec().rank(), dec().rank()) - .template triangularView<Upper>() - .solveInPlace(c.topRows(dec().rank())); + dec()._solve_impl(rhs(), dst); + } +}; +#endif // EIGEN_TEST_EVALUATORS - for(Index i = 0; i < dec().rank(); ++i) dst.row(dec().colsPermutation().indices().coeff(i)) = c.row(i); - for(Index i = dec().rank(); i < cols; ++i) dst.row(dec().colsPermutation().indices().coeff(i)).setZero(); +#ifdef EIGEN_TEST_EVALUATORS +template<typename DstXprType, typename MatrixType, typename Scalar> +struct Assignment<DstXprType, Inverse<FullPivHouseholderQR<MatrixType> >, internal::assign_op<Scalar>, Dense2Dense, Scalar> +{ + typedef FullPivHouseholderQR<MatrixType> QrType; + typedef Inverse<QrType> SrcXprType; + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<Scalar> &) + { + dst = src.nestedExpression().solve(MatrixType::Identity(src.rows(), src.cols())); } }; +#endif /** \ingroup QR_Module * @@ -534,6 +589,7 @@ struct solve_retval<FullPivHouseholderQR<_MatrixType>, Rhs> * * \tparam MatrixType type of underlying dense matrix */ +// #ifndef EIGEN_TEST_EVALUATORS template<typename MatrixType> struct FullPivHouseholderQRMatrixQReturnType : public ReturnByValue<FullPivHouseholderQRMatrixQReturnType<MatrixType> > { @@ -550,7 +606,7 @@ public: : m_qr(qr), m_hCoeffs(hCoeffs), m_rowsTranspositions(rowsTranspositions) - {} + {} template <typename ResultType> void evalTo(ResultType& result) const @@ -580,8 +636,8 @@ public: } } - Index rows() const { return m_qr.rows(); } - Index cols() const { return m_qr.rows(); } + Index rows() const { return m_qr.rows(); } + Index cols() const { return m_qr.rows(); } protected: typename MatrixType::Nested m_qr; @@ -589,6 +645,13 @@ protected: typename IntDiagSizeVectorType::Nested m_rowsTranspositions; }; +// #ifdef EIGEN_TEST_EVALUATORS +// template<typename MatrixType> +// struct evaluator<FullPivHouseholderQRMatrixQReturnType<MatrixType> > +// : public evaluator<ReturnByValue<FullPivHouseholderQRMatrixQReturnType<MatrixType> > > +// {}; +// #endif + } // end namespace internal template<typename MatrixType> diff --git a/Eigen/src/QR/HouseholderQR.h b/Eigen/src/QR/HouseholderQR.h index 352dbf3f0..8808e6c0d 100644 --- a/Eigen/src/QR/HouseholderQR.h +++ b/Eigen/src/QR/HouseholderQR.h @@ -117,6 +117,15 @@ template<typename _MatrixType> class HouseholderQR * Example: \include HouseholderQR_solve.cpp * Output: \verbinclude HouseholderQR_solve.out */ +#ifdef EIGEN_TEST_EVALUATORS + template<typename Rhs> + inline const Solve<HouseholderQR, Rhs> + solve(const MatrixBase<Rhs>& b) const + { + eigen_assert(m_isInitialized && "HouseholderQR is not initialized."); + return Solve<HouseholderQR, Rhs>(*this, b.derived()); + } +#else template<typename Rhs> inline const internal::solve_retval<HouseholderQR, Rhs> solve(const MatrixBase<Rhs>& b) const @@ -124,6 +133,7 @@ template<typename _MatrixType> class HouseholderQR eigen_assert(m_isInitialized && "HouseholderQR is not initialized."); return internal::solve_retval<HouseholderQR, Rhs>(*this, b.derived()); } +#endif /** This method returns an expression of the unitary matrix Q as a sequence of Householder transformations. * @@ -187,6 +197,12 @@ template<typename _MatrixType> class HouseholderQR * For advanced uses only. */ const HCoeffsType& hCoeffs() const { return m_hCoeffs; } + + #ifndef EIGEN_PARSED_BY_DOXYGEN + template<typename RhsType, typename DstType> + EIGEN_DEVICE_FUNC + void _solve_impl(const RhsType &rhs, DstType &dst) const; + #endif protected: MatrixType m_qr; @@ -308,6 +324,35 @@ struct householder_qr_inplace_blocked } }; +} // end namespace internal + +#ifndef EIGEN_PARSED_BY_DOXYGEN +template<typename _MatrixType> +template<typename RhsType, typename DstType> +void HouseholderQR<_MatrixType>::_solve_impl(const RhsType &rhs, DstType &dst) const +{ + const Index rank = (std::min)(rows(), cols()); + eigen_assert(rhs.rows() == rows()); + + typename RhsType::PlainObject c(rhs); + + // Note that the matrix Q = H_0^* H_1^*... so its inverse is Q^* = (H_0 H_1 ...)^T + c.applyOnTheLeft(householderSequence( + m_qr.leftCols(rank), + m_hCoeffs.head(rank)).transpose() + ); + + m_qr.topLeftCorner(rank, rank) + .template triangularView<Upper>() + .solveInPlace(c.topRows(rank)); + + dst.topRows(rank) = c.topRows(rank); + dst.bottomRows(cols()-rank).setZero(); +} +#endif + +namespace internal { + template<typename _MatrixType, typename Rhs> struct solve_retval<HouseholderQR<_MatrixType>, Rhs> : solve_retval_base<HouseholderQR<_MatrixType>, Rhs> @@ -316,25 +361,7 @@ struct solve_retval<HouseholderQR<_MatrixType>, Rhs> template<typename Dest> void evalTo(Dest& dst) const { - const Index rows = dec().rows(), cols = dec().cols(); - const Index rank = (std::min)(rows, cols); - eigen_assert(rhs().rows() == rows); - - typename Rhs::PlainObject c(rhs()); - - // Note that the matrix Q = H_0^* H_1^*... so its inverse is Q^* = (H_0 H_1 ...)^T - c.applyOnTheLeft(householderSequence( - dec().matrixQR().leftCols(rank), - dec().hCoeffs().head(rank)).transpose() - ); - - dec().matrixQR() - .topLeftCorner(rank, rank) - .template triangularView<Upper>() - .solveInPlace(c.topRows(rank)); - - dst.topRows(rank) = c.topRows(rank); - dst.bottomRows(cols-rank).setZero(); + dec()._solve_impl(rhs(), dst); } }; diff --git a/Eigen/src/SVD/JacobiSVD.h b/Eigen/src/SVD/JacobiSVD.h index 412daa746..d1b63b607 100644 --- a/Eigen/src/SVD/JacobiSVD.h +++ b/Eigen/src/SVD/JacobiSVD.h @@ -658,6 +658,16 @@ template<typename _MatrixType, int QRPreconditioner> class JacobiSVD * \note SVD solving is implicitly least-squares. Thus, this method serves both purposes of exact solving and least-squares solving. * In other words, the returned solution is guaranteed to minimize the Euclidean norm \f$ \Vert A x - b \Vert \f$. */ +#ifdef EIGEN_TEST_EVALUATORS + template<typename Rhs> + inline const Solve<JacobiSVD, Rhs> + solve(const MatrixBase<Rhs>& b) const + { + eigen_assert(m_isInitialized && "JacobiSVD is not initialized."); + eigen_assert(computeU() && computeV() && "JacobiSVD::solve() requires both unitaries U and V to be computed (thin unitaries suffice)."); + return Solve<JacobiSVD, Rhs>(*this, b.derived()); + } +#else template<typename Rhs> inline const internal::solve_retval<JacobiSVD, Rhs> solve(const MatrixBase<Rhs>& b) const @@ -666,6 +676,7 @@ template<typename _MatrixType, int QRPreconditioner> class JacobiSVD eigen_assert(computeU() && computeV() && "JacobiSVD::solve() requires both unitaries U and V to be computed (thin unitaries suffice)."); return internal::solve_retval<JacobiSVD, Rhs>(*this, b.derived()); } +#endif /** \returns the number of singular values that are not exactly 0 */ Index nonzeroSingularValues() const @@ -739,6 +750,12 @@ template<typename _MatrixType, int QRPreconditioner> class JacobiSVD inline Index rows() const { return m_rows; } inline Index cols() const { return m_cols; } + + #ifndef EIGEN_PARSED_BY_DOXYGEN + template<typename RhsType, typename DstType> + EIGEN_DEVICE_FUNC + void _solve_impl(const RhsType &rhs, DstType &dst) const; + #endif private: void allocate(Index rows, Index cols, unsigned int computationOptions); @@ -917,7 +934,27 @@ JacobiSVD<MatrixType, QRPreconditioner>::compute(const MatrixType& matrix, unsig return *this; } +#ifndef EIGEN_PARSED_BY_DOXYGEN +template<typename _MatrixType, int QRPreconditioner> +template<typename RhsType, typename DstType> +void JacobiSVD<_MatrixType,QRPreconditioner>::_solve_impl(const RhsType &rhs, DstType &dst) const +{ + eigen_assert(rhs.rows() == rows()); + + // A = U S V^* + // So A^{-1} = V S^{-1} U^* + + Matrix<Scalar, Dynamic, RhsType::ColsAtCompileTime, 0, _MatrixType::MaxRowsAtCompileTime, RhsType::MaxColsAtCompileTime> tmp; + Index l_rank = rank(); + + tmp.noalias() = m_matrixU.leftCols(l_rank).adjoint() * rhs; + tmp = m_singularValues.head(l_rank).asDiagonal().inverse() * tmp; + dst = m_matrixV.leftCols(l_rank) * tmp; +} +#endif + namespace internal { +#ifndef EIGEN_TEST_EVALUATORS template<typename _MatrixType, int QRPreconditioner, typename Rhs> struct solve_retval<JacobiSVD<_MatrixType, QRPreconditioner>, Rhs> : solve_retval_base<JacobiSVD<_MatrixType, QRPreconditioner>, Rhs> @@ -927,19 +964,10 @@ struct solve_retval<JacobiSVD<_MatrixType, QRPreconditioner>, Rhs> template<typename Dest> void evalTo(Dest& dst) const { - eigen_assert(rhs().rows() == dec().rows()); - - // A = U S V^* - // So A^{-1} = V S^{-1} U^* - - Matrix<Scalar, Dynamic, Rhs::ColsAtCompileTime, 0, _MatrixType::MaxRowsAtCompileTime, Rhs::MaxColsAtCompileTime> tmp; - Index rank = dec().rank(); - - tmp.noalias() = dec().matrixU().leftCols(rank).adjoint() * rhs(); - tmp = dec().singularValues().head(rank).asDiagonal().inverse() * tmp; - dst = dec().matrixV().leftCols(rank) * tmp; + dec()._solve_impl(rhs(), dst); } }; +#endif } // end namespace internal #ifndef __CUDACC__ diff --git a/Eigen/src/SparseCholesky/SimplicialCholesky.h b/Eigen/src/SparseCholesky/SimplicialCholesky.h index e1f96ba5a..1abd31304 100644 --- a/Eigen/src/SparseCholesky/SimplicialCholesky.h +++ b/Eigen/src/SparseCholesky/SimplicialCholesky.h @@ -255,8 +255,8 @@ template<typename _MatrixType, int _UpLo, typename _Ordering> struct traits<Simp typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::Index Index; typedef SparseMatrix<Scalar, ColMajor, Index> CholMatrixType; - typedef SparseTriangularView<CholMatrixType, Eigen::Lower> MatrixL; - typedef SparseTriangularView<typename CholMatrixType::AdjointReturnType, Eigen::Upper> MatrixU; + typedef TriangularView<CholMatrixType, Eigen::Lower> MatrixL; + typedef TriangularView<typename CholMatrixType::AdjointReturnType, Eigen::Upper> MatrixU; static inline MatrixL getL(const MatrixType& m) { return m; } static inline MatrixU getU(const MatrixType& m) { return m.adjoint(); } }; @@ -269,8 +269,8 @@ template<typename _MatrixType,int _UpLo, typename _Ordering> struct traits<Simpl typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::Index Index; typedef SparseMatrix<Scalar, ColMajor, Index> CholMatrixType; - typedef SparseTriangularView<CholMatrixType, Eigen::UnitLower> MatrixL; - typedef SparseTriangularView<typename CholMatrixType::AdjointReturnType, Eigen::UnitUpper> MatrixU; + typedef TriangularView<CholMatrixType, Eigen::UnitLower> MatrixL; + typedef TriangularView<typename CholMatrixType::AdjointReturnType, Eigen::UnitUpper> MatrixU; static inline MatrixL getL(const MatrixType& m) { return m; } static inline MatrixU getU(const MatrixType& m) { return m.adjoint(); } }; diff --git a/Eigen/src/SparseCore/ConservativeSparseSparseProduct.h b/Eigen/src/SparseCore/ConservativeSparseSparseProduct.h index 8067565f9..815fdb6d8 100644 --- a/Eigen/src/SparseCore/ConservativeSparseSparseProduct.h +++ b/Eigen/src/SparseCore/ConservativeSparseSparseProduct.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2011 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -36,6 +36,11 @@ static void conservative_sparse_sparse_product_impl(const Lhs& lhs, const Rhs& r // per column of the lhs. // Therefore, we have nnz(lhs*rhs) = nnz(lhs) + nnz(rhs) Index estimated_nnz_prod = lhs.nonZeros() + rhs.nonZeros(); + +#ifdef EIGEN_TEST_EVALUATORS + typename evaluator<Lhs>::type lhsEval(lhs); + typename evaluator<Rhs>::type rhsEval(rhs); +#endif res.setZero(); res.reserve(Index(estimated_nnz_prod)); @@ -45,11 +50,19 @@ static void conservative_sparse_sparse_product_impl(const Lhs& lhs, const Rhs& r res.startVec(j); Index nnz = 0; +#ifndef EIGEN_TEST_EVALUATORS for (typename Rhs::InnerIterator rhsIt(rhs, j); rhsIt; ++rhsIt) +#else + for (typename evaluator<Rhs>::InnerIterator rhsIt(rhsEval, j); rhsIt; ++rhsIt) +#endif { Scalar y = rhsIt.value(); Index k = rhsIt.index(); +#ifndef EIGEN_TEST_EVALUATORS for (typename Lhs::InnerIterator lhsIt(lhs, k); lhsIt; ++lhsIt) +#else + for (typename evaluator<Lhs>::InnerIterator lhsIt(lhsEval, k); lhsIt; ++lhsIt) +#endif { Index i = lhsIt.index(); Scalar x = lhsIt.value(); diff --git a/Eigen/src/SparseCore/MappedSparseMatrix.h b/Eigen/src/SparseCore/MappedSparseMatrix.h index ab1a266a9..9205b906f 100644 --- a/Eigen/src/SparseCore/MappedSparseMatrix.h +++ b/Eigen/src/SparseCore/MappedSparseMatrix.h @@ -176,6 +176,34 @@ class MappedSparseMatrix<Scalar,_Flags,_Index>::ReverseInnerIterator const Index m_end; }; +#ifdef EIGEN_ENABLE_EVALUATORS +namespace internal { + +template<typename _Scalar, int _Options, typename _Index> +struct evaluator<MappedSparseMatrix<_Scalar,_Options,_Index> > + : evaluator_base<MappedSparseMatrix<_Scalar,_Options,_Index> > +{ + typedef MappedSparseMatrix<_Scalar,_Options,_Index> MappedSparseMatrixType; + typedef typename MappedSparseMatrixType::InnerIterator InnerIterator; + typedef typename MappedSparseMatrixType::ReverseInnerIterator ReverseInnerIterator; + + enum { + CoeffReadCost = NumTraits<_Scalar>::ReadCost, + Flags = MappedSparseMatrixType::Flags + }; + + evaluator() : m_matrix(0) {} + evaluator(const MappedSparseMatrixType &mat) : m_matrix(&mat) {} + + operator MappedSparseMatrixType&() { return m_matrix->const_cast_derived(); } + operator const MappedSparseMatrixType&() const { return *m_matrix; } + + const MappedSparseMatrixType *m_matrix; +}; + +} +#endif + } // end namespace Eigen #endif // EIGEN_MAPPED_SPARSEMATRIX_H diff --git a/Eigen/src/SparseCore/SparseAssign.h b/Eigen/src/SparseCore/SparseAssign.h new file mode 100644 index 000000000..0a29cb32f --- /dev/null +++ b/Eigen/src/SparseCore/SparseAssign.h @@ -0,0 +1,293 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_SPARSEASSIGN_H +#define EIGEN_SPARSEASSIGN_H + +namespace Eigen { + +#ifndef EIGEN_TEST_EVALUATORS + +template<typename Derived> +template<typename OtherDerived> +Derived& SparseMatrixBase<Derived>::operator=(const EigenBase<OtherDerived> &other) +{ + other.derived().evalTo(derived()); + return derived(); +} + +template<typename Derived> +template<typename OtherDerived> +Derived& SparseMatrixBase<Derived>::operator=(const ReturnByValue<OtherDerived>& other) +{ + other.evalTo(derived()); + return derived(); +} + +template<typename Derived> +template<typename OtherDerived> +inline Derived& SparseMatrixBase<Derived>::operator=(const SparseMatrixBase<OtherDerived>& other) +{ + return assign(other.derived()); +} + +template<typename Derived> +inline Derived& SparseMatrixBase<Derived>::operator=(const Derived& other) +{ +// if (other.isRValue()) +// derived().swap(other.const_cast_derived()); +// else + return assign(other.derived()); +} + +template<typename Derived> +template<typename OtherDerived> +inline Derived& SparseMatrixBase<Derived>::assign(const OtherDerived& other) +{ + const bool transpose = (Flags & RowMajorBit) != (OtherDerived::Flags & RowMajorBit); + const Index outerSize = (int(OtherDerived::Flags) & RowMajorBit) ? Index(other.rows()) : Index(other.cols()); + if ((!transpose) && other.isRValue()) + { + // eval without temporary + derived().resize(Index(other.rows()), Index(other.cols())); + derived().setZero(); + derived().reserve((std::max)(this->rows(),this->cols())*2); + for (Index j=0; j<outerSize; ++j) + { + derived().startVec(j); + for (typename OtherDerived::InnerIterator it(other, typename OtherDerived::Index(j)); it; ++it) + { + Scalar v = it.value(); + derived().insertBackByOuterInner(j,Index(it.index())) = v; + } + } + derived().finalize(); + } + else + { + assignGeneric(other); + } + return derived(); +} + +template<typename Derived> +template<typename OtherDerived> +inline void SparseMatrixBase<Derived>::assignGeneric(const OtherDerived& other) +{ + //const bool transpose = (Flags & RowMajorBit) != (OtherDerived::Flags & RowMajorBit); + eigen_assert(( ((internal::traits<Derived>::SupportedAccessPatterns&OuterRandomAccessPattern)==OuterRandomAccessPattern) || + (!((Flags & RowMajorBit) != (OtherDerived::Flags & RowMajorBit)))) && + "the transpose operation is supposed to be handled in SparseMatrix::operator="); + + enum { Flip = (Flags & RowMajorBit) != (OtherDerived::Flags & RowMajorBit) }; + + const Index outerSize = Index(other.outerSize()); + //typedef typename internal::conditional<transpose, LinkedVectorMatrix<Scalar,Flags&RowMajorBit>, Derived>::type TempType; + // thanks to shallow copies, we always eval to a tempary + Derived temp(Index(other.rows()), Index(other.cols())); + + temp.reserve((std::max)(this->rows(),this->cols())*2); + for (Index j=0; j<outerSize; ++j) + { + temp.startVec(j); + for (typename OtherDerived::InnerIterator it(other.derived(), typename OtherDerived::Index(j)); it; ++it) + { + Scalar v = it.value(); + temp.insertBackByOuterInner(Flip?Index(it.index()):j,Flip?j:Index(it.index())) = v; + } + } + temp.finalize(); + + derived() = temp.markAsRValue(); +} + +// template<typename Lhs, typename Rhs> +// inline Derived& operator=(const SparseSparseProduct<Lhs,Rhs>& product); +// +// template<typename OtherDerived> +// Derived& operator+=(const SparseMatrixBase<OtherDerived>& other); +// template<typename OtherDerived> +// Derived& operator-=(const SparseMatrixBase<OtherDerived>& other); +// +// Derived& operator*=(const Scalar& other); +// Derived& operator/=(const Scalar& other); +// +// template<typename OtherDerived> +// Derived& operator*=(const SparseMatrixBase<OtherDerived>& other); + +#else // EIGEN_TEST_EVALUATORS + +template<typename Derived> +template<typename OtherDerived> +Derived& SparseMatrixBase<Derived>::operator=(const EigenBase<OtherDerived> &other) +{ + // TODO use the evaluator mechanism + other.derived().evalTo(derived()); + return derived(); +} + +template<typename Derived> +template<typename OtherDerived> +Derived& SparseMatrixBase<Derived>::operator=(const ReturnByValue<OtherDerived>& other) +{ + // TODO use the evaluator mechanism + other.evalTo(derived()); + return derived(); +} + +template<typename Derived> +template<typename OtherDerived> +inline Derived& SparseMatrixBase<Derived>::operator=(const SparseMatrixBase<OtherDerived>& other) +{ + // FIXME, by default sparse evaluation do not alias, so we should be able to bypass the generic call_assignment + internal::call_assignment/*_no_alias*/(derived(), other.derived()); + return derived(); +} + +template<typename Derived> +inline Derived& SparseMatrixBase<Derived>::operator=(const Derived& other) +{ + internal::call_assignment_no_alias(derived(), other.derived()); + return derived(); +} + +namespace internal { + +template<> +struct storage_kind_to_evaluator_kind<Sparse> { + typedef IteratorBased Kind; +}; + +template<> +struct storage_kind_to_shape<Sparse> { + typedef SparseShape Shape; +}; + +struct Sparse2Sparse {}; +struct Sparse2Dense {}; + +template<> struct AssignmentKind<SparseShape, SparseShape> { typedef Sparse2Sparse Kind; }; +template<> struct AssignmentKind<SparseShape, SparseTriangularShape> { typedef Sparse2Sparse Kind; }; +template<> struct AssignmentKind<DenseShape, SparseShape> { typedef Sparse2Dense Kind; }; + + +template<typename DstXprType, typename SrcXprType> +void assign_sparse_to_sparse(DstXprType &dst, const SrcXprType &src) +{ + eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); + + typedef typename DstXprType::Index Index; + typedef typename DstXprType::Scalar Scalar; + typedef typename internal::evaluator<DstXprType>::type DstEvaluatorType; + typedef typename internal::evaluator<SrcXprType>::type SrcEvaluatorType; + + SrcEvaluatorType srcEvaluator(src); + + const bool transpose = (DstEvaluatorType::Flags & RowMajorBit) != (SrcEvaluatorType::Flags & RowMajorBit); + const Index outerEvaluationSize = (SrcEvaluatorType::Flags&RowMajorBit) ? src.rows() : src.cols(); + if ((!transpose) && src.isRValue()) + { + // eval without temporary + dst.resize(src.rows(), src.cols()); + dst.setZero(); + dst.reserve((std::max)(src.rows(),src.cols())*2); + for (Index j=0; j<outerEvaluationSize; ++j) + { + dst.startVec(j); + for (typename SrcEvaluatorType::InnerIterator it(srcEvaluator, j); it; ++it) + { + Scalar v = it.value(); + dst.insertBackByOuterInner(j,it.index()) = v; + } + } + dst.finalize(); + } + else + { + // eval through a temporary + eigen_assert(( ((internal::traits<DstXprType>::SupportedAccessPatterns & OuterRandomAccessPattern)==OuterRandomAccessPattern) || + (!((DstEvaluatorType::Flags & RowMajorBit) != (SrcEvaluatorType::Flags & RowMajorBit)))) && + "the transpose operation is supposed to be handled in SparseMatrix::operator="); + + enum { Flip = (DstEvaluatorType::Flags & RowMajorBit) != (SrcEvaluatorType::Flags & RowMajorBit) }; + + + DstXprType temp(src.rows(), src.cols()); + + temp.reserve((std::max)(src.rows(),src.cols())*2); + for (Index j=0; j<outerEvaluationSize; ++j) + { + temp.startVec(j); + for (typename SrcEvaluatorType::InnerIterator it(srcEvaluator, j); it; ++it) + { + Scalar v = it.value(); + temp.insertBackByOuterInner(Flip?it.index():j,Flip?j:it.index()) = v; + } + } + temp.finalize(); + + dst = temp.markAsRValue(); + } +} + +// Generic Sparse to Sparse assignment +template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> +struct Assignment<DstXprType, SrcXprType, Functor, Sparse2Sparse, Scalar> +{ + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar> &/*func*/) + { + eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); + + assign_sparse_to_sparse(dst.derived(), src.derived()); + } +}; + +// Sparse to Dense assignment +template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar> +struct Assignment<DstXprType, SrcXprType, Functor, Sparse2Dense, Scalar> +{ + static void run(DstXprType &dst, const SrcXprType &src, const Functor &func) + { + eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); + typedef typename SrcXprType::Index Index; + + typename internal::evaluator<SrcXprType>::type srcEval(src); + typename internal::evaluator<DstXprType>::type dstEval(dst); + const Index outerEvaluationSize = (internal::evaluator<SrcXprType>::Flags&RowMajorBit) ? src.rows() : src.cols(); + for (Index j=0; j<outerEvaluationSize; ++j) + for (typename internal::evaluator<SrcXprType>::InnerIterator i(srcEval,j); i; ++i) + func.assignCoeff(dstEval.coeffRef(i.row(),i.col()), i.value()); + } +}; + +template< typename DstXprType, typename SrcXprType, typename Scalar> +struct Assignment<DstXprType, SrcXprType, internal::assign_op<typename DstXprType::Scalar>, Sparse2Dense, Scalar> +{ + static void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar> &) + { + eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); + typedef typename SrcXprType::Index Index; + + dst.setZero(); + typename internal::evaluator<SrcXprType>::type srcEval(src); + typename internal::evaluator<DstXprType>::type dstEval(dst); + const Index outerEvaluationSize = (internal::evaluator<SrcXprType>::Flags&RowMajorBit) ? src.rows() : src.cols(); + for (Index j=0; j<outerEvaluationSize; ++j) + for (typename internal::evaluator<SrcXprType>::InnerIterator i(srcEval,j); i; ++i) + dstEval.coeffRef(i.row(),i.col()) = i.value(); + } +}; + +} // end namespace internal + +#endif // EIGEN_TEST_EVALUATORS + +} // end namespace Eigen + +#endif // EIGEN_SPARSEASSIGN_H diff --git a/Eigen/src/SparseCore/SparseBlock.h b/Eigen/src/SparseCore/SparseBlock.h index 491cc72b0..21445b645 100644 --- a/Eigen/src/SparseCore/SparseBlock.h +++ b/Eigen/src/SparseCore/SparseBlock.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library
// for linear algebra.
//
-// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
+// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr>
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License v. 2.0. If a copy of the MPL was not distributed
@@ -12,6 +12,7 @@ namespace Eigen {
+// Subset of columns or rows
template<typename XprType, int BlockRows, int BlockCols>
class BlockImpl<XprType,BlockRows,BlockCols,true,Sparse>
: public SparseMatrixBase<Block<XprType,BlockRows,BlockCols,true> >
@@ -25,6 +26,7 @@ protected: public:
EIGEN_SPARSE_PUBLIC_INTERFACE(BlockType)
+#ifndef EIGEN_TEST_EVALUATORS
class InnerIterator: public XprType::InnerIterator
{
typedef typename BlockImpl::Index Index;
@@ -49,6 +51,7 @@ public: protected:
Index m_outer;
};
+#endif // EIGEN_TEST_EVALUATORS
inline BlockImpl(const XprType& xpr, Index i)
: m_matrix(xpr), m_outerStart(i), m_outerSize(OuterSize)
@@ -63,13 +66,30 @@ public: Index nonZeros() const
{
+#ifndef EIGEN_TEST_EVALUATORS
Index nnz = 0;
Index end = m_outerStart + m_outerSize.value();
for(Index j=m_outerStart; j<end; ++j)
for(typename XprType::InnerIterator it(m_matrix, j); it; ++it)
++nnz;
return nnz;
+#else // EIGEN_TEST_EVALUATORS
+ typedef typename internal::evaluator<XprType>::type EvaluatorType;
+ EvaluatorType matEval(m_matrix);
+ Index nnz = 0;
+ Index end = m_outerStart + m_outerSize.value();
+ for(int j=m_outerStart; j<end; ++j)
+ for(typename EvaluatorType::InnerIterator it(matEval, j); it; ++it)
+ ++nnz;
+ return nnz;
+#endif // EIGEN_TEST_EVALUATORS
}
+
+ inline const _MatrixTypeNested& nestedExpression() const { return m_matrix; }
+ Index startRow() const { return IsRowMajor ? m_outerStart : 0; }
+ Index startCol() const { return IsRowMajor ? 0 : m_outerStart; }
+ Index blockRows() const { return IsRowMajor ? m_outerSize.value() : m_matrix.rows(); }
+ Index blockCols() const { return IsRowMajor ? m_matrix.cols() : m_outerSize.value(); }
protected:
@@ -101,6 +121,7 @@ protected: enum { OuterSize = IsRowMajor ? BlockRows : BlockCols };
public:
+#ifndef EIGEN_TEST_EVALUATORS
class InnerIterator: public SparseMatrixType::InnerIterator
{
public:
@@ -123,6 +144,7 @@ public: protected:
Index m_outer;
};
+#endif // EIGEN_TEST_EVALUATORS
inline sparse_matrix_block_impl(const SparseMatrixType& xpr, Index i)
: m_matrix(xpr), m_outerStart(i), m_outerSize(OuterSize)
@@ -248,6 +270,12 @@ public: EIGEN_STRONG_INLINE Index rows() const { return IsRowMajor ? m_outerSize.value() : m_matrix.rows(); }
EIGEN_STRONG_INLINE Index cols() const { return IsRowMajor ? m_matrix.cols() : m_outerSize.value(); }
+
+ inline const _MatrixTypeNested& nestedExpression() const { return m_matrix; }
+ Index startRow() const { return IsRowMajor ? m_outerStart : 0; }
+ Index startCol() const { return IsRowMajor ? 0 : m_outerStart; }
+ Index blockRows() const { return IsRowMajor ? m_outerSize.value() : m_matrix.rows(); }
+ Index blockCols() const { return IsRowMajor ? m_matrix.cols() : m_outerSize.value(); }
protected:
@@ -406,8 +434,7 @@ public: m_startCol.value() + (RowsAtCompileTime == 1 ? index : 0));
}
- inline const _MatrixTypeNested& nestedExpression() const { return m_matrix; }
-
+#ifndef EIGEN_TEST_EVALUATORS
typedef internal::GenericSparseBlockInnerIteratorImpl<XprType,BlockRows,BlockCols,InnerPanel> InnerIterator;
class ReverseInnerIterator : public _MatrixTypeNested::ReverseInnerIterator
@@ -433,6 +460,14 @@ public: inline operator bool() const { return Base::operator bool() && Base::index() >= m_begin; }
};
+#endif // EIGEN_TEST_EVALUATORS
+
+ inline const _MatrixTypeNested& nestedExpression() const { return m_matrix; }
+ Index startRow() const { return m_startRow.value(); }
+ Index startCol() const { return m_startCol.value(); }
+ Index blockRows() const { return m_blockRows.value(); }
+ Index blockCols() const { return m_blockCols.value(); }
+
protected:
friend class internal::GenericSparseBlockInnerIteratorImpl<XprType,BlockRows,BlockCols,InnerPanel>;
friend class ReverseInnerIterator;
@@ -538,7 +573,126 @@ namespace internal { inline operator bool() const { return m_outerPos < m_end; }
};
+
+#ifdef EIGEN_TEST_EVALUATORS
+
+//
+template<typename ArgType, int BlockRows, int BlockCols, bool InnerPanel>
+struct unary_evaluator<Block<ArgType,BlockRows,BlockCols,InnerPanel>, IteratorBased >
+ : public evaluator_base<Block<ArgType,BlockRows,BlockCols,InnerPanel> >
+{
+ class InnerVectorInnerIterator;
+ class OuterVectorInnerIterator;
+ public:
+ typedef Block<ArgType,BlockRows,BlockCols,InnerPanel> XprType;
+ typedef typename XprType::Index Index;
+ typedef typename XprType::Scalar Scalar;
+
+ class ReverseInnerIterator;
+
+ enum {
+ IsRowMajor = XprType::IsRowMajor,
+
+ OuterVector = (BlockCols==1 && ArgType::IsRowMajor)
+ | // FIXME | instead of || to please GCC 4.4.0 stupid warning "suggest parentheses around &&".
+ // revert to || as soon as not needed anymore.
+ (BlockRows==1 && !ArgType::IsRowMajor),
+
+ CoeffReadCost = evaluator<ArgType>::CoeffReadCost,
+ Flags = XprType::Flags
+ };
+
+ typedef typename internal::conditional<OuterVector,OuterVectorInnerIterator,InnerVectorInnerIterator>::type InnerIterator;
+
+ unary_evaluator(const XprType& op)
+ : m_argImpl(op.nestedExpression()), m_block(op)
+ {}
+
+ protected:
+ typedef typename evaluator<ArgType>::InnerIterator EvalIterator;
+
+ typename evaluator<ArgType>::nestedType m_argImpl;
+ const XprType &m_block;
+};
+
+template<typename ArgType, int BlockRows, int BlockCols, bool InnerPanel>
+class unary_evaluator<Block<ArgType,BlockRows,BlockCols,InnerPanel>, IteratorBased>::InnerVectorInnerIterator
+ : public EvalIterator
+{
+ const XprType& m_block;
+ Index m_end;
+public:
+
+ EIGEN_STRONG_INLINE InnerVectorInnerIterator(const unary_evaluator& aEval, Index outer)
+ : EvalIterator(aEval.m_argImpl, outer + (IsRowMajor ? aEval.m_block.startRow() : aEval.m_block.startCol())),
+ m_block(aEval.m_block),
+ m_end(IsRowMajor ? aEval.m_block.startCol()+aEval.m_block.blockCols() : aEval.m_block.startRow()+aEval.m_block.blockRows())
+ {
+ while( (EvalIterator::operator bool()) && (EvalIterator::index() < (IsRowMajor ? m_block.startCol() : m_block.startRow())) )
+ EvalIterator::operator++();
+ }
+
+ inline Index index() const { return EvalIterator::index() - (IsRowMajor ? m_block.startCol() : m_block.startRow()); }
+ inline Index outer() const { return EvalIterator::outer() - (IsRowMajor ? m_block.startRow() : m_block.startCol()); }
+ inline Index row() const { return EvalIterator::row() - m_block.startRow(); }
+ inline Index col() const { return EvalIterator::col() - m_block.startCol(); }
+
+ inline operator bool() const { return EvalIterator::operator bool() && EvalIterator::index() < m_end; }
+};
+
+template<typename ArgType, int BlockRows, int BlockCols, bool InnerPanel>
+class unary_evaluator<Block<ArgType,BlockRows,BlockCols,InnerPanel>, IteratorBased>::OuterVectorInnerIterator
+{
+ const unary_evaluator& m_eval;
+ Index m_outerPos;
+ Index m_innerIndex;
+ Scalar m_value;
+ Index m_end;
+public:
+
+ EIGEN_STRONG_INLINE OuterVectorInnerIterator(const unary_evaluator& aEval, Index outer)
+ : m_eval(aEval),
+ m_outerPos( (IsRowMajor ? aEval.m_block.startCol() : aEval.m_block.startRow()) - 1), // -1 so that operator++ finds the first non-zero entry
+ m_innerIndex(IsRowMajor ? aEval.m_block.startRow() : aEval.m_block.startCol()),
+ m_end(IsRowMajor ? aEval.m_block.startCol()+aEval.m_block.blockCols() : aEval.m_block.startRow()+aEval.m_block.blockRows())
+ {
+ EIGEN_UNUSED_VARIABLE(outer);
+ eigen_assert(outer==0);
+
+ ++(*this);
+ }
+
+ inline Index index() const { return m_outerPos - (IsRowMajor ? m_eval.m_block.startCol() : m_eval.m_block.startRow()); }
+ inline Index outer() const { return 0; }
+ inline Index row() const { return IsRowMajor ? 0 : index(); }
+ inline Index col() const { return IsRowMajor ? index() : 0; }
+
+ inline Scalar value() const { return m_value; }
+
+ inline OuterVectorInnerIterator& operator++()
+ {
+ // search next non-zero entry
+ while(m_outerPos<m_end)
+ {
+ m_outerPos++;
+ EvalIterator it(m_eval.m_argImpl, m_outerPos);
+ // search for the key m_innerIndex in the current outer-vector
+ while(it && it.index() < m_innerIndex) ++it;
+ if(it && it.index()==m_innerIndex)
+ {
+ m_value = it.value();
+ break;
+ }
+ }
+ return *this;
+ }
+ inline operator bool() const { return m_outerPos < m_end; }
+};
+
+
+#endif // EIGEN_TEST_EVALUATORS
+
} // end namespace internal
diff --git a/Eigen/src/SparseCore/SparseCwiseBinaryOp.h b/Eigen/src/SparseCore/SparseCwiseBinaryOp.h index 60fdd214a..3a7e72cd2 100644 --- a/Eigen/src/SparseCore/SparseCwiseBinaryOp.h +++ b/Eigen/src/SparseCore/SparseCwiseBinaryOp.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -31,12 +31,6 @@ namespace Eigen { namespace internal { -template<> struct promote_storage_type<Dense,Sparse> -{ typedef Sparse ret; }; - -template<> struct promote_storage_type<Sparse,Dense> -{ typedef Sparse ret; }; - template<typename BinaryOp, typename Lhs, typename Rhs, typename Derived, typename _LhsStorageMode = typename traits<Lhs>::StorageKind, typename _RhsStorageMode = typename traits<Rhs>::StorageKind> @@ -44,6 +38,8 @@ class sparse_cwise_binary_op_inner_iterator_selector; } // end namespace internal +#ifndef EIGEN_TEST_EVALUATORS + template<typename BinaryOp, typename Lhs, typename Rhs> class CwiseBinaryOpImpl<BinaryOp, Lhs, Rhs, Sparse> : public SparseMatrixBase<CwiseBinaryOp<BinaryOp, Lhs, Rhs> > @@ -290,6 +286,314 @@ class sparse_cwise_binary_op_inner_iterator_selector<scalar_product_op<T>, Lhs, } // end namespace internal +#else // EIGEN_TEST_EVALUATORS + +namespace internal { + + +// Generic "sparse OP sparse" +template<typename BinaryOp, typename Lhs, typename Rhs> +struct binary_evaluator<CwiseBinaryOp<BinaryOp, Lhs, Rhs>, IteratorBased, IteratorBased> + : evaluator_base<CwiseBinaryOp<BinaryOp, Lhs, Rhs> > +{ +protected: + typedef typename evaluator<Lhs>::InnerIterator LhsIterator; + typedef typename evaluator<Rhs>::InnerIterator RhsIterator; +public: + typedef CwiseBinaryOp<BinaryOp, Lhs, Rhs> XprType; + + class ReverseInnerIterator; + class InnerIterator + { + typedef typename traits<XprType>::Scalar Scalar; + typedef typename XprType::Index Index; + + public: + + EIGEN_STRONG_INLINE InnerIterator(const binary_evaluator& aEval, Index outer) + : m_lhsIter(aEval.m_lhsImpl,outer), m_rhsIter(aEval.m_rhsImpl,outer), m_functor(aEval.m_functor) + { + this->operator++(); + } + + EIGEN_STRONG_INLINE InnerIterator& operator++() + { + if (m_lhsIter && m_rhsIter && (m_lhsIter.index() == m_rhsIter.index())) + { + m_id = m_lhsIter.index(); + m_value = m_functor(m_lhsIter.value(), m_rhsIter.value()); + ++m_lhsIter; + ++m_rhsIter; + } + else if (m_lhsIter && (!m_rhsIter || (m_lhsIter.index() < m_rhsIter.index()))) + { + m_id = m_lhsIter.index(); + m_value = m_functor(m_lhsIter.value(), Scalar(0)); + ++m_lhsIter; + } + else if (m_rhsIter && (!m_lhsIter || (m_lhsIter.index() > m_rhsIter.index()))) + { + m_id = m_rhsIter.index(); + m_value = m_functor(Scalar(0), m_rhsIter.value()); + ++m_rhsIter; + } + else + { + m_value = 0; // this is to avoid a compilation warning + m_id = -1; + } + return *this; + } + + EIGEN_STRONG_INLINE Scalar value() const { return m_value; } + + EIGEN_STRONG_INLINE Index index() const { return m_id; } + EIGEN_STRONG_INLINE Index row() const { return Lhs::IsRowMajor ? m_lhsIter.row() : index(); } + EIGEN_STRONG_INLINE Index col() const { return Lhs::IsRowMajor ? index() : m_lhsIter.col(); } + + EIGEN_STRONG_INLINE operator bool() const { return m_id>=0; } + + protected: + LhsIterator m_lhsIter; + RhsIterator m_rhsIter; + const BinaryOp& m_functor; + Scalar m_value; + Index m_id; + }; + + + enum { + CoeffReadCost = evaluator<Lhs>::CoeffReadCost + evaluator<Rhs>::CoeffReadCost + functor_traits<BinaryOp>::Cost, + Flags = XprType::Flags + }; + + binary_evaluator(const XprType& xpr) + : m_functor(xpr.functor()), + m_lhsImpl(xpr.lhs()), + m_rhsImpl(xpr.rhs()) + { } + +protected: + const BinaryOp m_functor; + typename evaluator<Lhs>::nestedType m_lhsImpl; + typename evaluator<Rhs>::nestedType m_rhsImpl; +}; + +// "sparse .* sparse" +template<typename T, typename Lhs, typename Rhs> +struct binary_evaluator<CwiseBinaryOp<scalar_product_op<T>, Lhs, Rhs>, IteratorBased, IteratorBased> + : evaluator_base<CwiseBinaryOp<scalar_product_op<T>, Lhs, Rhs> > +{ +protected: + typedef scalar_product_op<T> BinaryOp; + typedef typename evaluator<Lhs>::InnerIterator LhsIterator; + typedef typename evaluator<Rhs>::InnerIterator RhsIterator; +public: + typedef CwiseBinaryOp<BinaryOp, Lhs, Rhs> XprType; + + class ReverseInnerIterator; + class InnerIterator + { + typedef typename traits<XprType>::Scalar Scalar; + typedef typename XprType::Index Index; + + public: + + EIGEN_STRONG_INLINE InnerIterator(const binary_evaluator& aEval, Index outer) + : m_lhsIter(aEval.m_lhsImpl,outer), m_rhsIter(aEval.m_rhsImpl,outer), m_functor(aEval.m_functor) + { + while (m_lhsIter && m_rhsIter && (m_lhsIter.index() != m_rhsIter.index())) + { + if (m_lhsIter.index() < m_rhsIter.index()) + ++m_lhsIter; + else + ++m_rhsIter; + } + } + + EIGEN_STRONG_INLINE InnerIterator& operator++() + { + ++m_lhsIter; + ++m_rhsIter; + while (m_lhsIter && m_rhsIter && (m_lhsIter.index() != m_rhsIter.index())) + { + if (m_lhsIter.index() < m_rhsIter.index()) + ++m_lhsIter; + else + ++m_rhsIter; + } + return *this; + } + + EIGEN_STRONG_INLINE Scalar value() const { return m_functor(m_lhsIter.value(), m_rhsIter.value()); } + + EIGEN_STRONG_INLINE Index index() const { return m_lhsIter.index(); } + EIGEN_STRONG_INLINE Index row() const { return m_lhsIter.row(); } + EIGEN_STRONG_INLINE Index col() const { return m_lhsIter.col(); } + + EIGEN_STRONG_INLINE operator bool() const { return (m_lhsIter && m_rhsIter); } + + protected: + LhsIterator m_lhsIter; + RhsIterator m_rhsIter; + const BinaryOp& m_functor; + }; + + + enum { + CoeffReadCost = evaluator<Lhs>::CoeffReadCost + evaluator<Rhs>::CoeffReadCost + functor_traits<BinaryOp>::Cost, + Flags = XprType::Flags + }; + + binary_evaluator(const XprType& xpr) + : m_functor(xpr.functor()), + m_lhsImpl(xpr.lhs()), + m_rhsImpl(xpr.rhs()) + { } + +protected: + const BinaryOp m_functor; + typename evaluator<Lhs>::nestedType m_lhsImpl; + typename evaluator<Rhs>::nestedType m_rhsImpl; +}; + +// "dense .* sparse" +template<typename T, typename Lhs, typename Rhs> +struct binary_evaluator<CwiseBinaryOp<scalar_product_op<T>, Lhs, Rhs>, IndexBased, IteratorBased> + : evaluator_base<CwiseBinaryOp<scalar_product_op<T>, Lhs, Rhs> > +{ +protected: + typedef scalar_product_op<T> BinaryOp; + typedef typename evaluator<Lhs>::type LhsEvaluator; + typedef typename evaluator<Rhs>::InnerIterator RhsIterator; +public: + typedef CwiseBinaryOp<BinaryOp, Lhs, Rhs> XprType; + + class ReverseInnerIterator; + class InnerIterator + { + typedef typename traits<XprType>::Scalar Scalar; + typedef typename XprType::Index Index; + enum { IsRowMajor = (int(Rhs::Flags)&RowMajorBit)==RowMajorBit }; + + public: + + EIGEN_STRONG_INLINE InnerIterator(const binary_evaluator& aEval, Index outer) + : m_lhsEval(aEval.m_lhsImpl), m_rhsIter(aEval.m_rhsImpl,outer), m_functor(aEval.m_functor), m_outer(outer) + {} + + EIGEN_STRONG_INLINE InnerIterator& operator++() + { + ++m_rhsIter; + return *this; + } + + EIGEN_STRONG_INLINE Scalar value() const + { return m_functor(m_lhsEval.coeff(IsRowMajor?m_outer:m_rhsIter.index(),IsRowMajor?m_rhsIter.index():m_outer), m_rhsIter.value()); } + + EIGEN_STRONG_INLINE Index index() const { return m_rhsIter.index(); } + EIGEN_STRONG_INLINE Index row() const { return m_rhsIter.row(); } + EIGEN_STRONG_INLINE Index col() const { return m_rhsIter.col(); } + + EIGEN_STRONG_INLINE operator bool() const { return m_rhsIter; } + + protected: + const LhsEvaluator &m_lhsEval; + RhsIterator m_rhsIter; + const BinaryOp& m_functor; + const Index m_outer; + }; + + + enum { + CoeffReadCost = evaluator<Lhs>::CoeffReadCost + evaluator<Rhs>::CoeffReadCost + functor_traits<BinaryOp>::Cost, + Flags = XprType::Flags + }; + + binary_evaluator(const XprType& xpr) + : m_functor(xpr.functor()), + m_lhsImpl(xpr.lhs()), + m_rhsImpl(xpr.rhs()) + { } + +protected: + const BinaryOp m_functor; + typename evaluator<Lhs>::nestedType m_lhsImpl; + typename evaluator<Rhs>::nestedType m_rhsImpl; +}; + +// "sparse .* dense" +template<typename T, typename Lhs, typename Rhs> +struct binary_evaluator<CwiseBinaryOp<scalar_product_op<T>, Lhs, Rhs>, IteratorBased, IndexBased> + : evaluator_base<CwiseBinaryOp<scalar_product_op<T>, Lhs, Rhs> > +{ +protected: + typedef scalar_product_op<T> BinaryOp; + typedef typename evaluator<Lhs>::InnerIterator LhsIterator; + typedef typename evaluator<Rhs>::type RhsEvaluator; +public: + typedef CwiseBinaryOp<BinaryOp, Lhs, Rhs> XprType; + + class ReverseInnerIterator; + class InnerIterator + { + typedef typename traits<XprType>::Scalar Scalar; + typedef typename XprType::Index Index; + enum { IsRowMajor = (int(Lhs::Flags)&RowMajorBit)==RowMajorBit }; + + public: + + EIGEN_STRONG_INLINE InnerIterator(const binary_evaluator& aEval, Index outer) + : m_lhsIter(aEval.m_lhsImpl,outer), m_rhsEval(aEval.m_rhsImpl), m_functor(aEval.m_functor), m_outer(outer) + {} + + EIGEN_STRONG_INLINE InnerIterator& operator++() + { + ++m_lhsIter; + return *this; + } + + EIGEN_STRONG_INLINE Scalar value() const + { return m_functor(m_lhsIter.value(), + m_rhsEval.coeff(IsRowMajor?m_outer:m_lhsIter.index(),IsRowMajor?m_lhsIter.index():m_outer)); } + + EIGEN_STRONG_INLINE Index index() const { return m_lhsIter.index(); } + EIGEN_STRONG_INLINE Index row() const { return m_lhsIter.row(); } + EIGEN_STRONG_INLINE Index col() const { return m_lhsIter.col(); } + + EIGEN_STRONG_INLINE operator bool() const { return m_lhsIter; } + + protected: + LhsIterator m_lhsIter; + const RhsEvaluator &m_rhsEval; + const BinaryOp& m_functor; + const Index m_outer; + }; + + + enum { + CoeffReadCost = evaluator<Lhs>::CoeffReadCost + evaluator<Rhs>::CoeffReadCost + functor_traits<BinaryOp>::Cost, + Flags = XprType::Flags + }; + + binary_evaluator(const XprType& xpr) + : m_functor(xpr.functor()), + m_lhsImpl(xpr.lhs()), + m_rhsImpl(xpr.rhs()) + { } + +protected: + const BinaryOp m_functor; + typename evaluator<Lhs>::nestedType m_lhsImpl; + typename evaluator<Rhs>::nestedType m_rhsImpl; +}; + +} + +#endif + + + /*************************************************************************** * Implementation of SparseMatrixBase and SparseCwise functions/operators ***************************************************************************/ diff --git a/Eigen/src/SparseCore/SparseCwiseUnaryOp.h b/Eigen/src/SparseCore/SparseCwiseUnaryOp.h index 5a50c7803..0e0f9f5f5 100644 --- a/Eigen/src/SparseCore/SparseCwiseUnaryOp.h +++ b/Eigen/src/SparseCore/SparseCwiseUnaryOp.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2010 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -11,6 +11,8 @@ #define EIGEN_SPARSE_CWISE_UNARY_OP_H namespace Eigen { + +#ifndef EIGEN_TEST_EVALUATORS template<typename UnaryOp, typename MatrixType> class CwiseUnaryOpImpl<UnaryOp,MatrixType,Sparse> @@ -18,12 +20,12 @@ class CwiseUnaryOpImpl<UnaryOp,MatrixType,Sparse> { public: - class InnerIterator; - class ReverseInnerIterator; - typedef CwiseUnaryOp<UnaryOp, MatrixType> Derived; EIGEN_SPARSE_PUBLIC_INTERFACE(Derived) + class InnerIterator; + class ReverseInnerIterator; + protected: typedef typename internal::traits<Derived>::_XprTypeNested _MatrixTypeNested; typedef typename _MatrixTypeNested::InnerIterator MatrixTypeIterator; @@ -138,6 +140,159 @@ class CwiseUnaryViewImpl<ViewOp,MatrixType,Sparse>::ReverseInnerIterator const ViewOp m_functor; }; +#else // EIGEN_TEST_EVALUATORS + +namespace internal { + +template<typename UnaryOp, typename ArgType> +struct unary_evaluator<CwiseUnaryOp<UnaryOp,ArgType>, IteratorBased> + : public evaluator_base<CwiseUnaryOp<UnaryOp,ArgType> > +{ + public: + typedef CwiseUnaryOp<UnaryOp, ArgType> XprType; + + class InnerIterator; +// class ReverseInnerIterator; + + enum { + CoeffReadCost = evaluator<ArgType>::CoeffReadCost + functor_traits<UnaryOp>::Cost, + Flags = XprType::Flags + }; + + unary_evaluator(const XprType& op) : m_functor(op.functor()), m_argImpl(op.nestedExpression()) {} + + protected: + typedef typename evaluator<ArgType>::InnerIterator EvalIterator; +// typedef typename evaluator<ArgType>::ReverseInnerIterator EvalReverseIterator; + + const UnaryOp m_functor; + typename evaluator<ArgType>::nestedType m_argImpl; +}; + +template<typename UnaryOp, typename ArgType> +class unary_evaluator<CwiseUnaryOp<UnaryOp,ArgType>, IteratorBased>::InnerIterator + : public unary_evaluator<CwiseUnaryOp<UnaryOp,ArgType>, IteratorBased>::EvalIterator +{ + typedef typename XprType::Scalar Scalar; + typedef typename unary_evaluator<CwiseUnaryOp<UnaryOp,ArgType>, IteratorBased>::EvalIterator Base; + public: + + EIGEN_STRONG_INLINE InnerIterator(const unary_evaluator& unaryOp, typename XprType::Index outer) + : Base(unaryOp.m_argImpl,outer), m_functor(unaryOp.m_functor) + {} + + EIGEN_STRONG_INLINE InnerIterator& operator++() + { Base::operator++(); return *this; } + + EIGEN_STRONG_INLINE Scalar value() const { return m_functor(Base::value()); } + + protected: + const UnaryOp m_functor; + private: + Scalar& valueRef(); +}; + +// template<typename UnaryOp, typename ArgType> +// class unary_evaluator<CwiseUnaryOp<UnaryOp,ArgType>, IteratorBased>::ReverseInnerIterator +// : public unary_evaluator<CwiseUnaryOp<UnaryOp,ArgType>, IteratorBased>::EvalReverseIterator +// { +// typedef typename XprType::Scalar Scalar; +// typedef typename unary_evaluator<CwiseUnaryOp<UnaryOp,ArgType>, IteratorBased>::EvalReverseIterator Base; +// public: +// +// EIGEN_STRONG_INLINE ReverseInnerIterator(const XprType& unaryOp, typename XprType::Index outer) +// : Base(unaryOp.derived().nestedExpression(),outer), m_functor(unaryOp.derived().functor()) +// {} +// +// EIGEN_STRONG_INLINE ReverseInnerIterator& operator--() +// { Base::operator--(); return *this; } +// +// EIGEN_STRONG_INLINE Scalar value() const { return m_functor(Base::value()); } +// +// protected: +// const UnaryOp m_functor; +// private: +// Scalar& valueRef(); +// }; + + + + + +template<typename ViewOp, typename ArgType> +struct unary_evaluator<CwiseUnaryView<ViewOp,ArgType>, IteratorBased> + : public evaluator_base<CwiseUnaryView<ViewOp,ArgType> > +{ + public: + typedef CwiseUnaryView<ViewOp, ArgType> XprType; + + class InnerIterator; + class ReverseInnerIterator; + + enum { + CoeffReadCost = evaluator<ArgType>::CoeffReadCost + functor_traits<ViewOp>::Cost, + Flags = XprType::Flags + }; + + unary_evaluator(const XprType& op) : m_functor(op.functor()), m_argImpl(op.nestedExpression()) {} + + protected: + typedef typename evaluator<ArgType>::InnerIterator EvalIterator; +// typedef typename evaluator<ArgType>::ReverseInnerIterator EvalReverseIterator; + + const ViewOp m_functor; + typename evaluator<ArgType>::nestedType m_argImpl; +}; + +template<typename ViewOp, typename ArgType> +class unary_evaluator<CwiseUnaryView<ViewOp,ArgType>, IteratorBased>::InnerIterator + : public unary_evaluator<CwiseUnaryView<ViewOp,ArgType>, IteratorBased>::EvalIterator +{ + typedef typename XprType::Scalar Scalar; + typedef typename unary_evaluator<CwiseUnaryView<ViewOp,ArgType>, IteratorBased>::EvalIterator Base; + public: + + EIGEN_STRONG_INLINE InnerIterator(const unary_evaluator& unaryOp, typename XprType::Index outer) + : Base(unaryOp.m_argImpl,outer), m_functor(unaryOp.m_functor) + {} + + EIGEN_STRONG_INLINE InnerIterator& operator++() + { Base::operator++(); return *this; } + + EIGEN_STRONG_INLINE Scalar value() const { return m_functor(Base::value()); } + EIGEN_STRONG_INLINE Scalar& valueRef() { return m_functor(Base::valueRef()); } + + protected: + const ViewOp m_functor; +}; + +// template<typename ViewOp, typename ArgType> +// class unary_evaluator<CwiseUnaryView<ViewOp,ArgType>, IteratorBased>::ReverseInnerIterator +// : public unary_evaluator<CwiseUnaryView<ViewOp,ArgType>, IteratorBased>::EvalReverseIterator +// { +// typedef typename XprType::Scalar Scalar; +// typedef typename unary_evaluator<CwiseUnaryView<ViewOp,ArgType>, IteratorBased>::EvalReverseIterator Base; +// public: +// +// EIGEN_STRONG_INLINE ReverseInnerIterator(const XprType& unaryOp, typename XprType::Index outer) +// : Base(unaryOp.derived().nestedExpression(),outer), m_functor(unaryOp.derived().functor()) +// {} +// +// EIGEN_STRONG_INLINE ReverseInnerIterator& operator--() +// { Base::operator--(); return *this; } +// +// EIGEN_STRONG_INLINE Scalar value() const { return m_functor(Base::value()); } +// EIGEN_STRONG_INLINE Scalar& valueRef() { return m_functor(Base::valueRef()); } +// +// protected: +// const ViewOp m_functor; +// }; + + +} // end namespace internal + +#endif // EIGEN_TEST_EVALUATORS + template<typename Derived> EIGEN_STRONG_INLINE Derived& SparseMatrixBase<Derived>::operator*=(const Scalar& other) diff --git a/Eigen/src/SparseCore/SparseDenseProduct.h b/Eigen/src/SparseCore/SparseDenseProduct.h index d40e966c1..a715b8bde 100644 --- a/Eigen/src/SparseCore/SparseDenseProduct.h +++ b/Eigen/src/SparseCore/SparseDenseProduct.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2010 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -12,6 +12,155 @@ namespace Eigen { +namespace internal { + +template <> struct product_promote_storage_type<Sparse,Dense, OuterProduct> { typedef Sparse ret; }; +template <> struct product_promote_storage_type<Dense,Sparse, OuterProduct> { typedef Sparse ret; }; + +template<typename SparseLhsType, typename DenseRhsType, typename DenseResType, + typename AlphaType, + int LhsStorageOrder = ((SparseLhsType::Flags&RowMajorBit)==RowMajorBit) ? RowMajor : ColMajor, + bool ColPerCol = ((DenseRhsType::Flags&RowMajorBit)==0) || DenseRhsType::ColsAtCompileTime==1> +struct sparse_time_dense_product_impl; + +template<typename SparseLhsType, typename DenseRhsType, typename DenseResType> +struct sparse_time_dense_product_impl<SparseLhsType,DenseRhsType,DenseResType, typename DenseResType::Scalar, RowMajor, true> +{ + typedef typename internal::remove_all<SparseLhsType>::type Lhs; + typedef typename internal::remove_all<DenseRhsType>::type Rhs; + typedef typename internal::remove_all<DenseResType>::type Res; + typedef typename Lhs::Index Index; +#ifndef EIGEN_TEST_EVALUATORS + typedef typename Lhs::InnerIterator LhsInnerIterator; +#else + typedef typename evaluator<Lhs>::InnerIterator LhsInnerIterator; +#endif + static void run(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const typename Res::Scalar& alpha) + { +#ifndef EIGEN_TEST_EVALUATORS + const Lhs &lhsEval(lhs); +#else + typename evaluator<Lhs>::type lhsEval(lhs); +#endif + for(Index c=0; c<rhs.cols(); ++c) + { + Index n = lhs.outerSize(); + for(Index j=0; j<n; ++j) + { + typename Res::Scalar tmp(0); + for(LhsInnerIterator it(lhsEval,j); it ;++it) + tmp += it.value() * rhs.coeff(it.index(),c); + res.coeffRef(j,c) = alpha * tmp; + } + } + } +}; + +template<typename T1, typename T2/*, int _Options, typename _StrideType*/> +struct scalar_product_traits<T1, Ref<T2/*, _Options, _StrideType*/> > +{ + enum { + Defined = 1 + }; + typedef typename CwiseUnaryOp<scalar_multiple2_op<T1, typename T2::Scalar>, T2>::PlainObject ReturnType; +}; +template<typename SparseLhsType, typename DenseRhsType, typename DenseResType, typename AlphaType> +struct sparse_time_dense_product_impl<SparseLhsType,DenseRhsType,DenseResType, AlphaType, ColMajor, true> +{ + typedef typename internal::remove_all<SparseLhsType>::type Lhs; + typedef typename internal::remove_all<DenseRhsType>::type Rhs; + typedef typename internal::remove_all<DenseResType>::type Res; + typedef typename Lhs::Index Index; +#ifndef EIGEN_TEST_EVALUATORS + typedef typename Lhs::InnerIterator LhsInnerIterator; +#else + typedef typename evaluator<Lhs>::InnerIterator LhsInnerIterator; +#endif + static void run(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const AlphaType& alpha) + { +#ifndef EIGEN_TEST_EVALUATORS + const Lhs &lhsEval(lhs); +#else + typename evaluator<Lhs>::type lhsEval(lhs); +#endif + for(Index c=0; c<rhs.cols(); ++c) + { + for(Index j=0; j<lhs.outerSize(); ++j) + { +// typename Res::Scalar rhs_j = alpha * rhs.coeff(j,c); + typename internal::scalar_product_traits<AlphaType, typename Rhs::Scalar>::ReturnType rhs_j(alpha * rhs.coeff(j,c)); + for(LhsInnerIterator it(lhsEval,j); it ;++it) + res.coeffRef(it.index(),c) += it.value() * rhs_j; + } + } + } +}; + +template<typename SparseLhsType, typename DenseRhsType, typename DenseResType> +struct sparse_time_dense_product_impl<SparseLhsType,DenseRhsType,DenseResType, typename DenseResType::Scalar, RowMajor, false> +{ + typedef typename internal::remove_all<SparseLhsType>::type Lhs; + typedef typename internal::remove_all<DenseRhsType>::type Rhs; + typedef typename internal::remove_all<DenseResType>::type Res; + typedef typename Lhs::Index Index; +#ifndef EIGEN_TEST_EVALUATORS + typedef typename Lhs::InnerIterator LhsInnerIterator; +#else + typedef typename evaluator<Lhs>::InnerIterator LhsInnerIterator; +#endif + static void run(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const typename Res::Scalar& alpha) + { +#ifndef EIGEN_TEST_EVALUATORS + const Lhs &lhsEval(lhs); +#else + typename evaluator<Lhs>::type lhsEval(lhs); +#endif + for(Index j=0; j<lhs.outerSize(); ++j) + { + typename Res::RowXpr res_j(res.row(j)); + for(LhsInnerIterator it(lhsEval,j); it ;++it) + res_j += (alpha*it.value()) * rhs.row(it.index()); + } + } +}; + +template<typename SparseLhsType, typename DenseRhsType, typename DenseResType> +struct sparse_time_dense_product_impl<SparseLhsType,DenseRhsType,DenseResType, typename DenseResType::Scalar, ColMajor, false> +{ + typedef typename internal::remove_all<SparseLhsType>::type Lhs; + typedef typename internal::remove_all<DenseRhsType>::type Rhs; + typedef typename internal::remove_all<DenseResType>::type Res; + typedef typename Lhs::Index Index; +#ifndef EIGEN_TEST_EVALUATORS + typedef typename Lhs::InnerIterator LhsInnerIterator; +#else + typedef typename evaluator<Lhs>::InnerIterator LhsInnerIterator; +#endif + static void run(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const typename Res::Scalar& alpha) + { +#ifndef EIGEN_TEST_EVALUATORS + const Lhs &lhsEval(lhs); +#else + typename evaluator<Lhs>::type lhsEval(lhs); +#endif + for(Index j=0; j<lhs.outerSize(); ++j) + { + typename Rhs::ConstRowXpr rhs_j(rhs.row(j)); + for(LhsInnerIterator it(lhsEval,j); it ;++it) + res.row(it.index()) += (alpha*it.value()) * rhs_j; + } + } +}; + +template<typename SparseLhsType, typename DenseRhsType, typename DenseResType,typename AlphaType> +inline void sparse_time_dense_product(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const AlphaType& alpha) +{ + sparse_time_dense_product_impl<SparseLhsType,DenseRhsType,DenseResType, AlphaType>::run(lhs, rhs, res, alpha); +} + +} // end namespace internal + +#ifndef EIGEN_TEST_EVALUATORS template<typename Lhs, typename Rhs, int InnerSize> struct SparseDenseProductReturnType { typedef SparseTimeDenseProduct<Lhs,Rhs> Type; @@ -159,111 +308,6 @@ struct traits<SparseTimeDenseProduct<Lhs,Rhs> > typedef MatrixXpr XprKind; }; -template<typename SparseLhsType, typename DenseRhsType, typename DenseResType, - typename AlphaType, - int LhsStorageOrder = ((SparseLhsType::Flags&RowMajorBit)==RowMajorBit) ? RowMajor : ColMajor, - bool ColPerCol = ((DenseRhsType::Flags&RowMajorBit)==0) || DenseRhsType::ColsAtCompileTime==1> -struct sparse_time_dense_product_impl; - -template<typename SparseLhsType, typename DenseRhsType, typename DenseResType> -struct sparse_time_dense_product_impl<SparseLhsType,DenseRhsType,DenseResType, typename DenseResType::Scalar, RowMajor, true> -{ - typedef typename internal::remove_all<SparseLhsType>::type Lhs; - typedef typename internal::remove_all<DenseRhsType>::type Rhs; - typedef typename internal::remove_all<DenseResType>::type Res; - typedef typename Lhs::Index Index; - typedef typename Lhs::InnerIterator LhsInnerIterator; - static void run(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const typename Res::Scalar& alpha) - { - for(Index c=0; c<rhs.cols(); ++c) - { - Index n = lhs.outerSize(); - for(Index j=0; j<n; ++j) - { - typename Res::Scalar tmp(0); - for(LhsInnerIterator it(lhs,j); it ;++it) - tmp += it.value() * rhs.coeff(it.index(),c); - res.coeffRef(j,c) = alpha * tmp; - } - } - } -}; - -template<typename T1, typename T2/*, int _Options, typename _StrideType*/> -struct scalar_product_traits<T1, Ref<T2/*, _Options, _StrideType*/> > -{ - enum { - Defined = 1 - }; - typedef typename CwiseUnaryOp<scalar_multiple2_op<T1, typename T2::Scalar>, T2>::PlainObject ReturnType; -}; -template<typename SparseLhsType, typename DenseRhsType, typename DenseResType, typename AlphaType> -struct sparse_time_dense_product_impl<SparseLhsType,DenseRhsType,DenseResType, AlphaType, ColMajor, true> -{ - typedef typename internal::remove_all<SparseLhsType>::type Lhs; - typedef typename internal::remove_all<DenseRhsType>::type Rhs; - typedef typename internal::remove_all<DenseResType>::type Res; - typedef typename Lhs::InnerIterator LhsInnerIterator; - typedef typename Lhs::Index Index; - static void run(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const AlphaType& alpha) - { - for(Index c=0; c<rhs.cols(); ++c) - { - for(Index j=0; j<lhs.outerSize(); ++j) - { -// typename Res::Scalar rhs_j = alpha * rhs.coeff(j,c); - typename internal::scalar_product_traits<AlphaType, typename Rhs::Scalar>::ReturnType rhs_j(alpha * rhs.coeff(j,c)); - for(LhsInnerIterator it(lhs,j); it ;++it) - res.coeffRef(it.index(),c) += it.value() * rhs_j; - } - } - } -}; - -template<typename SparseLhsType, typename DenseRhsType, typename DenseResType> -struct sparse_time_dense_product_impl<SparseLhsType,DenseRhsType,DenseResType, typename DenseResType::Scalar, RowMajor, false> -{ - typedef typename internal::remove_all<SparseLhsType>::type Lhs; - typedef typename internal::remove_all<DenseRhsType>::type Rhs; - typedef typename internal::remove_all<DenseResType>::type Res; - typedef typename Lhs::InnerIterator LhsInnerIterator; - typedef typename Lhs::Index Index; - static void run(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const typename Res::Scalar& alpha) - { - for(Index j=0; j<lhs.outerSize(); ++j) - { - typename Res::RowXpr res_j(res.row(j)); - for(LhsInnerIterator it(lhs,j); it ;++it) - res_j += (alpha*it.value()) * rhs.row(it.index()); - } - } -}; - -template<typename SparseLhsType, typename DenseRhsType, typename DenseResType> -struct sparse_time_dense_product_impl<SparseLhsType,DenseRhsType,DenseResType, typename DenseResType::Scalar, ColMajor, false> -{ - typedef typename internal::remove_all<SparseLhsType>::type Lhs; - typedef typename internal::remove_all<DenseRhsType>::type Rhs; - typedef typename internal::remove_all<DenseResType>::type Res; - typedef typename Lhs::InnerIterator LhsInnerIterator; - typedef typename Lhs::Index Index; - static void run(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const typename Res::Scalar& alpha) - { - for(Index j=0; j<lhs.outerSize(); ++j) - { - typename Rhs::ConstRowXpr rhs_j(rhs.row(j)); - for(LhsInnerIterator it(lhs,j); it ;++it) - res.row(it.index()) += (alpha*it.value()) * rhs_j; - } - } -}; - -template<typename SparseLhsType, typename DenseRhsType, typename DenseResType,typename AlphaType> -inline void sparse_time_dense_product(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const AlphaType& alpha) -{ - sparse_time_dense_product_impl<SparseLhsType,DenseRhsType,DenseResType, AlphaType>::run(lhs, rhs, res, alpha); -} - } // end namespace internal template<typename Lhs, typename Rhs> @@ -318,6 +362,169 @@ class DenseTimeSparseProduct DenseTimeSparseProduct& operator=(const DenseTimeSparseProduct&); }; +// sparse * dense +template<typename Derived> +template<typename OtherDerived> +inline const typename SparseDenseProductReturnType<Derived,OtherDerived>::Type +SparseMatrixBase<Derived>::operator*(const MatrixBase<OtherDerived> &other) const +{ + return typename SparseDenseProductReturnType<Derived,OtherDerived>::Type(derived(), other.derived()); +} +#endif // EIGEN_TEST_EVALUATORS + +#ifdef EIGEN_TEST_EVALUATORS + +namespace internal { + +template<typename Lhs, typename Rhs, int ProductType> +struct generic_product_impl<Lhs, Rhs, SparseShape, DenseShape, ProductType> +{ + template<typename Dest> + static void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) + { + typedef typename nested_eval<Lhs,Dynamic>::type LhsNested; + typedef typename nested_eval<Rhs,Dynamic>::type RhsNested; + LhsNested lhsNested(lhs); + RhsNested rhsNested(rhs); + + dst.setZero(); + internal::sparse_time_dense_product(lhsNested, rhsNested, dst, typename Dest::Scalar(1)); + } +}; + +template<typename Lhs, typename Rhs, int ProductType> +struct generic_product_impl<Lhs, Rhs, DenseShape, SparseShape, ProductType> +{ + template<typename Dest> + static void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) + { + typedef typename nested_eval<Lhs,Dynamic>::type LhsNested; + typedef typename nested_eval<Rhs,Dynamic>::type RhsNested; + LhsNested lhsNested(lhs); + RhsNested rhsNested(rhs); + + dst.setZero(); + // transpoe everything + Transpose<Dest> dstT(dst); + internal::sparse_time_dense_product(rhsNested.transpose(), lhsNested.transpose(), dstT, typename Dest::Scalar(1)); + } +}; + +template<typename LhsT, typename RhsT, bool Transpose> +struct sparse_dense_outer_product_evaluator +{ +protected: + typedef typename conditional<Transpose,RhsT,LhsT>::type Lhs1; + typedef typename conditional<Transpose,LhsT,RhsT>::type Rhs; + typedef Product<LhsT,RhsT> ProdXprType; + + // if the actual left-hand side is a dense vector, + // then build a sparse-view so that we can seamlessly iterator over it. + typedef typename conditional<is_same<typename internal::traits<Lhs1>::StorageKind,Sparse>::value, + Lhs1, SparseView<Lhs1> >::type Lhs; + typedef typename conditional<is_same<typename internal::traits<Lhs1>::StorageKind,Sparse>::value, + Lhs1 const&, SparseView<Lhs1> >::type LhsArg; + + typedef typename evaluator<Lhs>::type LhsEval; + typedef typename evaluator<Rhs>::type RhsEval; + typedef typename evaluator<Lhs>::InnerIterator LhsIterator; + typedef typename ProdXprType::Scalar Scalar; + typedef typename ProdXprType::Index Index; + +public: + enum { + Flags = Transpose ? RowMajorBit : 0, + CoeffReadCost = Dynamic + }; + + class InnerIterator : public LhsIterator + { + public: + InnerIterator(const sparse_dense_outer_product_evaluator &xprEval, Index outer) + : LhsIterator(xprEval.m_lhsXprImpl, 0), + m_outer(outer), + m_empty(false), + m_factor(get(xprEval.m_rhsXprImpl, outer, typename internal::traits<Rhs>::StorageKind() )) + {} + + EIGEN_STRONG_INLINE Index outer() const { return m_outer; } + EIGEN_STRONG_INLINE Index row() const { return Transpose ? m_outer : LhsIterator::index(); } + EIGEN_STRONG_INLINE Index col() const { return Transpose ? LhsIterator::index() : m_outer; } + + EIGEN_STRONG_INLINE Scalar value() const { return LhsIterator::value() * m_factor; } + EIGEN_STRONG_INLINE operator bool() const { return LhsIterator::operator bool() && (!m_empty); } + + + protected: + Scalar get(const RhsEval &rhs, Index outer, Dense = Dense()) const + { + return rhs.coeff(outer); + } + + Scalar get(const RhsEval &rhs, Index outer, Sparse = Sparse()) + { + typename RhsEval::InnerIterator it(rhs, outer); + if (it && it.index()==0 && it.value()!=Scalar(0)) + return it.value(); + m_empty = true; + return Scalar(0); + } + + Index m_outer; + bool m_empty; + Scalar m_factor; + }; + + sparse_dense_outer_product_evaluator(const Lhs &lhs, const Rhs &rhs) + : m_lhs(lhs), m_lhsXprImpl(m_lhs), m_rhsXprImpl(rhs) + {} + + // transpose case + sparse_dense_outer_product_evaluator(const Rhs &rhs, const Lhs1 &lhs) + : m_lhs(lhs), m_lhsXprImpl(m_lhs), m_rhsXprImpl(rhs) + {} + +protected: + const LhsArg m_lhs; + typename evaluator<Lhs>::nestedType m_lhsXprImpl; + typename evaluator<Rhs>::nestedType m_rhsXprImpl; +}; + +// sparse * dense outer product +template<typename Lhs, typename Rhs> +struct product_evaluator<Product<Lhs, Rhs, DefaultProduct>, OuterProduct, SparseShape, DenseShape, typename traits<Lhs>::Scalar, typename traits<Rhs>::Scalar> + : sparse_dense_outer_product_evaluator<Lhs,Rhs, Lhs::IsRowMajor> +{ + typedef sparse_dense_outer_product_evaluator<Lhs,Rhs, Lhs::IsRowMajor> Base; + + typedef Product<Lhs, Rhs> XprType; + typedef typename XprType::PlainObject PlainObject; + + product_evaluator(const XprType& xpr) + : Base(xpr.lhs(), xpr.rhs()) + {} + +}; + +template<typename Lhs, typename Rhs> +struct product_evaluator<Product<Lhs, Rhs, DefaultProduct>, OuterProduct, DenseShape, SparseShape, typename traits<Lhs>::Scalar, typename traits<Rhs>::Scalar> + : sparse_dense_outer_product_evaluator<Lhs,Rhs, Rhs::IsRowMajor> +{ + typedef sparse_dense_outer_product_evaluator<Lhs,Rhs, Rhs::IsRowMajor> Base; + + typedef Product<Lhs, Rhs> XprType; + typedef typename XprType::PlainObject PlainObject; + + product_evaluator(const XprType& xpr) + : Base(xpr.lhs(), xpr.rhs()) + {} + +}; + +} // end namespace internal + +#endif // EIGEN_TEST_EVALUATORS + } // end namespace Eigen #endif // EIGEN_SPARSEDENSEPRODUCT_H diff --git a/Eigen/src/SparseCore/SparseDiagonalProduct.h b/Eigen/src/SparseCore/SparseDiagonalProduct.h index c056b4914..9f465a828 100644 --- a/Eigen/src/SparseCore/SparseDiagonalProduct.h +++ b/Eigen/src/SparseCore/SparseDiagonalProduct.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2009-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -24,8 +24,10 @@ namespace Eigen { // for that particular case // The two other cases are symmetric. +#ifndef EIGEN_TEST_EVALUATORS + namespace internal { - + template<typename Lhs, typename Rhs> struct traits<SparseDiagonalProduct<Lhs, Rhs> > { @@ -102,9 +104,14 @@ class SparseDiagonalProduct LhsNested m_lhs; RhsNested m_rhs; }; +#endif namespace internal { +#ifndef EIGEN_TEST_EVALUATORS + + + template<typename Lhs, typename Rhs, typename SparseDiagonalProductType> class sparse_diagonal_product_inner_iterator_selector <Lhs,Rhs,SparseDiagonalProductType,SDP_IsDiagonal,SDP_IsSparseRowMajor> @@ -181,10 +188,129 @@ class sparse_diagonal_product_inner_iterator_selector inline Index row() const { return m_outer; } }; +#else // EIGEN_TEST_EVALUATORS +enum { + SDP_AsScalarProduct, + SDP_AsCwiseProduct +}; + +template<typename SparseXprType, typename DiagonalCoeffType, int SDP_Tag> +struct sparse_diagonal_product_evaluator; + +template<typename Lhs, typename Rhs, int ProductTag> +struct product_evaluator<Product<Lhs, Rhs, DefaultProduct>, ProductTag, DiagonalShape, SparseShape, typename traits<Lhs>::Scalar, typename traits<Rhs>::Scalar> + : public sparse_diagonal_product_evaluator<Rhs, typename Lhs::DiagonalVectorType, Rhs::Flags&RowMajorBit?SDP_AsScalarProduct:SDP_AsCwiseProduct> +{ + typedef Product<Lhs, Rhs, DefaultProduct> XprType; + typedef evaluator<XprType> type; + typedef evaluator<XprType> nestedType; + enum { CoeffReadCost = Dynamic, Flags = Rhs::Flags&RowMajorBit }; // FIXME CoeffReadCost & Flags + + typedef sparse_diagonal_product_evaluator<Rhs, typename Lhs::DiagonalVectorType, Rhs::Flags&RowMajorBit?SDP_AsScalarProduct:SDP_AsCwiseProduct> Base; + product_evaluator(const XprType& xpr) : Base(xpr.rhs(), xpr.lhs().diagonal()) {} +}; + +template<typename Lhs, typename Rhs, int ProductTag> +struct product_evaluator<Product<Lhs, Rhs, DefaultProduct>, ProductTag, SparseShape, DiagonalShape, typename traits<Lhs>::Scalar, typename traits<Rhs>::Scalar> + : public sparse_diagonal_product_evaluator<Lhs, Transpose<const typename Rhs::DiagonalVectorType>, Lhs::Flags&RowMajorBit?SDP_AsCwiseProduct:SDP_AsScalarProduct> +{ + typedef Product<Lhs, Rhs, DefaultProduct> XprType; + typedef evaluator<XprType> type; + typedef evaluator<XprType> nestedType; + enum { CoeffReadCost = Dynamic, Flags = Lhs::Flags&RowMajorBit }; // FIXME CoeffReadCost & Flags + + typedef sparse_diagonal_product_evaluator<Lhs, Transpose<const typename Rhs::DiagonalVectorType>, Lhs::Flags&RowMajorBit?SDP_AsCwiseProduct:SDP_AsScalarProduct> Base; + product_evaluator(const XprType& xpr) : Base(xpr.lhs(), xpr.rhs().diagonal()) {} +}; + +template<typename SparseXprType, typename DiagonalCoeffType> +struct sparse_diagonal_product_evaluator<SparseXprType, DiagonalCoeffType, SDP_AsScalarProduct> +{ +protected: + typedef typename evaluator<SparseXprType>::InnerIterator SparseXprInnerIterator; + typedef typename SparseXprType::Scalar Scalar; + typedef typename SparseXprType::Index Index; + +public: + class InnerIterator : public SparseXprInnerIterator + { + public: + InnerIterator(const sparse_diagonal_product_evaluator &xprEval, Index outer) + : SparseXprInnerIterator(xprEval.m_sparseXprImpl, outer), + m_coeff(xprEval.m_diagCoeffImpl.coeff(outer)) + {} + + EIGEN_STRONG_INLINE Scalar value() const { return m_coeff * SparseXprInnerIterator::value(); } + protected: + typename DiagonalCoeffType::Scalar m_coeff; + }; + + sparse_diagonal_product_evaluator(const SparseXprType &sparseXpr, const DiagonalCoeffType &diagCoeff) + : m_sparseXprImpl(sparseXpr), m_diagCoeffImpl(diagCoeff) + {} + +protected: + typename evaluator<SparseXprType>::nestedType m_sparseXprImpl; + typename evaluator<DiagonalCoeffType>::nestedType m_diagCoeffImpl; +}; + + +template<typename SparseXprType, typename DiagCoeffType> +struct sparse_diagonal_product_evaluator<SparseXprType, DiagCoeffType, SDP_AsCwiseProduct> +{ + typedef typename SparseXprType::Scalar Scalar; + typedef typename SparseXprType::Index Index; + + typedef CwiseBinaryOp<scalar_product_op<Scalar>, + const typename SparseXprType::ConstInnerVectorReturnType, + const DiagCoeffType> CwiseProductType; + + typedef typename evaluator<CwiseProductType>::type CwiseProductEval; + typedef typename evaluator<CwiseProductType>::InnerIterator CwiseProductIterator; + + class InnerIterator + { + public: + InnerIterator(const sparse_diagonal_product_evaluator &xprEval, Index outer) + : m_cwiseEval(xprEval.m_sparseXprNested.innerVector(outer).cwiseProduct(xprEval.m_diagCoeffNested)), + m_cwiseIter(m_cwiseEval, 0), + m_outer(outer) + {} + + inline Scalar value() const { return m_cwiseIter.value(); } + inline Index index() const { return m_cwiseIter.index(); } + inline Index outer() const { return m_outer; } + inline Index col() const { return SparseXprType::IsRowMajor ? m_cwiseIter.index() : m_outer; } + inline Index row() const { return SparseXprType::IsRowMajor ? m_outer : m_cwiseIter.index(); } + + EIGEN_STRONG_INLINE InnerIterator& operator++() + { ++m_cwiseIter; return *this; } + inline operator bool() const { return m_cwiseIter; } + + protected: + CwiseProductEval m_cwiseEval; + CwiseProductIterator m_cwiseIter; + Index m_outer; + }; + + sparse_diagonal_product_evaluator(const SparseXprType &sparseXpr, const DiagCoeffType &diagCoeff) + : m_sparseXprNested(sparseXpr), m_diagCoeffNested(diagCoeff) + {} + +protected: + typename nested_eval<SparseXprType,1>::type m_sparseXprNested; + typename nested_eval<DiagCoeffType,SparseXprType::IsRowMajor ? SparseXprType::RowsAtCompileTime + : SparseXprType::ColsAtCompileTime>::type m_diagCoeffNested; +}; + +#endif // EIGEN_TEST_EVALUATORS + + } // end namespace internal -// SparseMatrixBase functions +#ifndef EIGEN_TEST_EVALUATORS +// SparseMatrixBase functions template<typename Derived> template<typename OtherDerived> const SparseDiagonalProduct<Derived,OtherDerived> @@ -192,6 +318,7 @@ SparseMatrixBase<Derived>::operator*(const DiagonalBase<OtherDerived> &other) co { return SparseDiagonalProduct<Derived,OtherDerived>(this->derived(), other.derived()); } +#endif // EIGEN_TEST_EVALUATORS } // end namespace Eigen diff --git a/Eigen/src/SparseCore/SparseDot.h b/Eigen/src/SparseCore/SparseDot.h index db39c9aec..a63cb003c 100644 --- a/Eigen/src/SparseCore/SparseDot.h +++ b/Eigen/src/SparseCore/SparseDot.h @@ -26,7 +26,12 @@ SparseMatrixBase<Derived>::dot(const MatrixBase<OtherDerived>& other) const eigen_assert(size() == other.size()); eigen_assert(other.size()>0 && "you are using a non initialized vector"); +#ifndef EIGEN_TEST_EVALUATORS typename Derived::InnerIterator i(derived(),0); +#else + typename internal::evaluator<Derived>::type thisEval(derived()); + typename internal::evaluator<Derived>::InnerIterator i(thisEval, 0); +#endif Scalar res(0); while (i) { @@ -49,6 +54,7 @@ SparseMatrixBase<Derived>::dot(const SparseMatrixBase<OtherDerived>& other) cons eigen_assert(size() == other.size()); +#ifndef EIGEN_TEST_EVALUATORS typedef typename Derived::Nested Nested; typedef typename OtherDerived::Nested OtherNested; typedef typename internal::remove_all<Nested>::type NestedCleaned; @@ -59,6 +65,13 @@ SparseMatrixBase<Derived>::dot(const SparseMatrixBase<OtherDerived>& other) cons typename NestedCleaned::InnerIterator i(nthis,0); typename OtherNestedCleaned::InnerIterator j(nother,0); +#else + typename internal::evaluator<Derived>::type thisEval(derived()); + typename internal::evaluator<Derived>::InnerIterator i(thisEval, 0); + + typename internal::evaluator<OtherDerived>::type otherEval(other.derived()); + typename internal::evaluator<OtherDerived>::InnerIterator j(otherEval, 0); +#endif Scalar res(0); while (i && j) { diff --git a/Eigen/src/SparseCore/SparseMatrix.h b/Eigen/src/SparseCore/SparseMatrix.h index 2ed2f3ebd..9e2dae1b3 100644 --- a/Eigen/src/SparseCore/SparseMatrix.h +++ b/Eigen/src/SparseCore/SparseMatrix.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2010 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -52,7 +52,9 @@ struct traits<SparseMatrix<_Scalar, _Options, _Index> > MaxRowsAtCompileTime = Dynamic, MaxColsAtCompileTime = Dynamic, Flags = _Options | NestByRefBit | LvalueBit, +#ifndef EIGEN_TEST_EVALUATORS CoeffReadCost = NumTraits<Scalar>::ReadCost, +#endif SupportedAccessPatterns = InnerRandomAccessPattern }; }; @@ -74,8 +76,10 @@ struct traits<Diagonal<const SparseMatrix<_Scalar, _Options, _Index>, DiagIndex> ColsAtCompileTime = 1, MaxRowsAtCompileTime = Dynamic, MaxColsAtCompileTime = 1, - Flags = 0, - CoeffReadCost = _MatrixTypeNested::CoeffReadCost*10 + Flags = 0 +#ifndef EIGEN_TEST_EVALUATORS + , CoeffReadCost = _MatrixTypeNested::CoeffReadCost*10 +#endif }; }; @@ -649,7 +653,13 @@ class SparseMatrix EIGEN_STATIC_ASSERT((internal::is_same<Scalar, typename OtherDerived::Scalar>::value), YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) check_template_parameters(); +#ifndef EIGEN_TEST_EVALUATORS *this = other.derived(); +#else + const bool needToTranspose = (Flags & RowMajorBit) != (internal::evaluator<OtherDerived>::Flags & RowMajorBit); + if (needToTranspose) *this = other.derived(); + else internal::call_assignment_no_alias(*this, other.derived()); +#endif } /** Constructs a sparse matrix from the sparse selfadjoint view \a other */ @@ -658,7 +668,11 @@ class SparseMatrix : m_outerSize(0), m_innerSize(0), m_outerIndex(0), m_innerNonZeros(0) { check_template_parameters(); +#ifndef EIGEN_TEST_EVALUATORS *this = other; +#else + Base::operator=(other); +#endif } /** Copy constructor (it performs a deep copy) */ @@ -722,7 +736,8 @@ class SparseMatrix return *this; } - #ifndef EIGEN_PARSED_BY_DOXYGEN +#ifndef EIGEN_PARSED_BY_DOXYGEN +#ifndef EIGEN_TEST_EVALUATORS template<typename Lhs, typename Rhs> inline SparseMatrix& operator=(const SparseSparseProduct<Lhs,Rhs>& product) { return Base::operator=(product); } @@ -733,11 +748,12 @@ class SparseMatrix initAssignment(other); return Base::operator=(other.derived()); } - +#endif // EIGEN_TEST_EVALUATORS + template<typename OtherDerived> inline SparseMatrix& operator=(const EigenBase<OtherDerived>& other) { return Base::operator=(other.derived()); } - #endif +#endif // EIGEN_PARSED_BY_DOXYGEN template<typename OtherDerived> EIGEN_DONT_INLINE SparseMatrix& operator=(const SparseMatrixBase<OtherDerived>& other); @@ -1055,6 +1071,7 @@ void SparseMatrix<Scalar,_Options,_Index>::sumupDuplicates() m_data.resize(m_outerIndex[m_outerSize]); } +#ifndef EIGEN_TEST_EVALUATORS template<typename Scalar, int _Options, typename _Index> template<typename OtherDerived> EIGEN_DONT_INLINE SparseMatrix<Scalar,_Options,_Index>& SparseMatrix<Scalar,_Options,_Index>::operator=(const SparseMatrixBase<OtherDerived>& other) @@ -1116,6 +1133,73 @@ EIGEN_DONT_INLINE SparseMatrix<Scalar,_Options,_Index>& SparseMatrix<Scalar,_Opt return Base::operator=(other.derived()); } } +#else +template<typename Scalar, int _Options, typename _Index> +template<typename OtherDerived> +EIGEN_DONT_INLINE SparseMatrix<Scalar,_Options,_Index>& SparseMatrix<Scalar,_Options,_Index>::operator=(const SparseMatrixBase<OtherDerived>& other) +{ + EIGEN_STATIC_ASSERT((internal::is_same<Scalar, typename OtherDerived::Scalar>::value), + YOU_MIXED_DIFFERENT_NUMERIC_TYPES__YOU_NEED_TO_USE_THE_CAST_METHOD_OF_MATRIXBASE_TO_CAST_NUMERIC_TYPES_EXPLICITLY) + + const bool needToTranspose = (Flags & RowMajorBit) != (internal::evaluator<OtherDerived>::Flags & RowMajorBit); + if (needToTranspose) + { + // two passes algorithm: + // 1 - compute the number of coeffs per dest inner vector + // 2 - do the actual copy/eval + // Since each coeff of the rhs has to be evaluated twice, let's evaluate it if needed + typedef typename internal::nested_eval<OtherDerived,2,typename internal::plain_matrix_type<OtherDerived>::type >::type OtherCopy; + typedef typename internal::remove_all<OtherCopy>::type _OtherCopy; + typedef internal::evaluator<_OtherCopy> OtherCopyEval; + OtherCopy otherCopy(other.derived()); + OtherCopyEval otherCopyEval(otherCopy); + + SparseMatrix dest(other.rows(),other.cols()); + Eigen::Map<Matrix<Index, Dynamic, 1> > (dest.m_outerIndex,dest.outerSize()).setZero(); + + // pass 1 + // FIXME the above copy could be merged with that pass + for (Index j=0; j<otherCopy.outerSize(); ++j) + for (typename OtherCopyEval::InnerIterator it(otherCopyEval, j); it; ++it) + ++dest.m_outerIndex[it.index()]; + + // prefix sum + Index count = 0; + Matrix<Index,Dynamic,1> positions(dest.outerSize()); + for (Index j=0; j<dest.outerSize(); ++j) + { + Index tmp = dest.m_outerIndex[j]; + dest.m_outerIndex[j] = count; + positions[j] = count; + count += tmp; + } + dest.m_outerIndex[dest.outerSize()] = count; + // alloc + dest.m_data.resize(count); + // pass 2 + for (Index j=0; j<otherCopy.outerSize(); ++j) + { + for (typename OtherCopyEval::InnerIterator it(otherCopyEval, j); it; ++it) + { + Index pos = positions[it.index()]++; + dest.m_data.index(pos) = j; + dest.m_data.value(pos) = it.value(); + } + } + this->swap(dest); + return *this; + } + else + { + if(other.isRValue()) + { + initAssignment(other.derived()); + } + // there is no special optimization + return Base::operator=(other.derived()); + } +} +#endif template<typename _Scalar, int _Options, typename _Index> EIGEN_DONT_INLINE typename SparseMatrix<_Scalar,_Options,_Index>::Scalar& SparseMatrix<_Scalar,_Options,_Index>::insertUncompressed(Index row, Index col) @@ -1256,6 +1340,38 @@ EIGEN_DONT_INLINE typename SparseMatrix<_Scalar,_Options,_Index>::Scalar& Sparse return (m_data.value(p) = 0); } +#ifdef EIGEN_ENABLE_EVALUATORS +namespace internal { + +template<typename _Scalar, int _Options, typename _Index> +struct evaluator<SparseMatrix<_Scalar,_Options,_Index> > + : evaluator_base<SparseMatrix<_Scalar,_Options,_Index> > +{ + typedef _Scalar Scalar; + typedef _Index Index; + typedef SparseMatrix<_Scalar,_Options,_Index> SparseMatrixType; + typedef typename SparseMatrixType::InnerIterator InnerIterator; + typedef typename SparseMatrixType::ReverseInnerIterator ReverseInnerIterator; + + enum { + CoeffReadCost = NumTraits<_Scalar>::ReadCost, + Flags = SparseMatrixType::Flags + }; + + evaluator() : m_matrix(0) {} + evaluator(const SparseMatrixType &mat) : m_matrix(&mat) {} + + operator SparseMatrixType&() { return m_matrix->const_cast_derived(); } + operator const SparseMatrixType&() const { return *m_matrix; } + + Scalar coeff(Index row, Index col) const { return m_matrix->coeff(row,col); } + + const SparseMatrixType *m_matrix; +}; + +} +#endif + } // end namespace Eigen #endif // EIGEN_SPARSEMATRIX_H diff --git a/Eigen/src/SparseCore/SparseMatrixBase.h b/Eigen/src/SparseCore/SparseMatrixBase.h index fb5025049..aff4c8b6f 100644 --- a/Eigen/src/SparseCore/SparseMatrixBase.h +++ b/Eigen/src/SparseCore/SparseMatrixBase.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2011 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -39,11 +39,7 @@ template<typename Derived> class SparseMatrixBase : public EigenBase<Derived> typedef EigenBase<Derived> Base; template<typename OtherDerived> - Derived& operator=(const EigenBase<OtherDerived> &other) - { - other.derived().evalTo(derived()); - return derived(); - } + Derived& operator=(const EigenBase<OtherDerived> &other); enum { @@ -83,10 +79,12 @@ template<typename Derived> class SparseMatrixBase : public EigenBase<Derived> * constructed from this one. See the \ref flags "list of flags". */ +#ifndef EIGEN_TEST_EVALUATORS CoeffReadCost = internal::traits<Derived>::CoeffReadCost, /**< This is a rough measure of how expensive it is to read one coefficient from * this expression. */ +#endif IsRowMajor = Flags&RowMajorBit ? 1 : 0, @@ -104,10 +102,9 @@ template<typename Derived> class SparseMatrixBase : public EigenBase<Derived> Transpose<const Derived> >::type AdjointReturnType; - + // FIXME storage order do not match evaluator storage order typedef SparseMatrix<Scalar, Flags&RowMajorBit ? RowMajor : ColMajor, Index> PlainObject; - #ifndef EIGEN_PARSED_BY_DOXYGEN /** This is the "real scalar" type; if the \a Scalar type is already real numbers * (e.g. int, float or double) then \a RealScalar is just the same as \a Scalar. If @@ -175,92 +172,27 @@ template<typename Derived> class SparseMatrixBase : public EigenBase<Derived> template<typename OtherDerived> - Derived& operator=(const ReturnByValue<OtherDerived>& other) - { - other.evalTo(derived()); - return derived(); - } - + Derived& operator=(const ReturnByValue<OtherDerived>& other); template<typename OtherDerived> - inline Derived& operator=(const SparseMatrixBase<OtherDerived>& other) - { - return assign(other.derived()); - } + inline Derived& operator=(const SparseMatrixBase<OtherDerived>& other); - inline Derived& operator=(const Derived& other) - { -// if (other.isRValue()) -// derived().swap(other.const_cast_derived()); -// else - return assign(other.derived()); - } + inline Derived& operator=(const Derived& other); protected: template<typename OtherDerived> - inline Derived& assign(const OtherDerived& other) - { - const bool transpose = (Flags & RowMajorBit) != (OtherDerived::Flags & RowMajorBit); - const Index outerSize = (int(OtherDerived::Flags) & RowMajorBit) ? Index(other.rows()) : Index(other.cols()); - if ((!transpose) && other.isRValue()) - { - // eval without temporary - derived().resize(Index(other.rows()), Index(other.cols())); - derived().setZero(); - derived().reserve((std::max)(this->rows(),this->cols())*2); - for (Index j=0; j<outerSize; ++j) - { - derived().startVec(j); - for (typename OtherDerived::InnerIterator it(other, typename OtherDerived::Index(j)); it; ++it) - { - Scalar v = it.value(); - derived().insertBackByOuterInner(j,Index(it.index())) = v; - } - } - derived().finalize(); - } - else - { - assignGeneric(other); - } - return derived(); - } + inline Derived& assign(const OtherDerived& other); template<typename OtherDerived> - inline void assignGeneric(const OtherDerived& other) - { - //const bool transpose = (Flags & RowMajorBit) != (OtherDerived::Flags & RowMajorBit); - eigen_assert(( ((internal::traits<Derived>::SupportedAccessPatterns&OuterRandomAccessPattern)==OuterRandomAccessPattern) || - (!((Flags & RowMajorBit) != (OtherDerived::Flags & RowMajorBit)))) && - "the transpose operation is supposed to be handled in SparseMatrix::operator="); - - enum { Flip = (Flags & RowMajorBit) != (OtherDerived::Flags & RowMajorBit) }; - - const Index outerSize = Index(other.outerSize()); - //typedef typename internal::conditional<transpose, LinkedVectorMatrix<Scalar,Flags&RowMajorBit>, Derived>::type TempType; - // thanks to shallow copies, we always eval to a tempary - Derived temp(Index(other.rows()), Index(other.cols())); - - temp.reserve((std::max)(this->rows(),this->cols())*2); - for (Index j=0; j<outerSize; ++j) - { - temp.startVec(j); - for (typename OtherDerived::InnerIterator it(other.derived(), typename OtherDerived::Index(j)); it; ++it) - { - Scalar v = it.value(); - temp.insertBackByOuterInner(Flip?Index(it.index()):j,Flip?j:Index(it.index())) = v; - } - } - temp.finalize(); - - derived() = temp.markAsRValue(); - } + inline void assignGeneric(const OtherDerived& other); public: +#ifndef EIGEN_TEST_EVALUATORS template<typename Lhs, typename Rhs> inline Derived& operator=(const SparseSparseProduct<Lhs,Rhs>& product); +#endif friend std::ostream & operator << (std::ostream & s, const SparseMatrixBase& m) { @@ -333,11 +265,12 @@ template<typename Derived> class SparseMatrixBase : public EigenBase<Derived> EIGEN_STRONG_INLINE const EIGEN_SPARSE_CWISE_PRODUCT_RETURN_TYPE cwiseProduct(const MatrixBase<OtherDerived> &other) const; +#ifndef EIGEN_TEST_EVALUATORS // sparse * sparse template<typename OtherDerived> const typename SparseSparseProductReturnType<Derived,OtherDerived>::Type operator*(const SparseMatrixBase<OtherDerived> &other) const; - + // sparse * diagonal template<typename OtherDerived> const SparseDiagonalProduct<Derived,OtherDerived> @@ -348,7 +281,7 @@ template<typename Derived> class SparseMatrixBase : public EigenBase<Derived> const SparseDiagonalProduct<OtherDerived,Derived> operator*(const DiagonalBase<OtherDerived> &lhs, const SparseMatrixBase& rhs) { return SparseDiagonalProduct<OtherDerived,Derived>(lhs.derived(), rhs.derived()); } - + /** dense * sparse (return a dense object unless it is an outer product) */ template<typename OtherDerived> friend const typename DenseSparseProductReturnType<OtherDerived,Derived>::Type @@ -361,6 +294,37 @@ template<typename Derived> class SparseMatrixBase : public EigenBase<Derived> operator*(const MatrixBase<OtherDerived> &other) const { return typename SparseDenseProductReturnType<Derived,OtherDerived>::Type(derived(), other.derived()); } +#else // EIGEN_TEST_EVALUATORS + // sparse * diagonal + template<typename OtherDerived> + const Product<Derived,OtherDerived> + operator*(const DiagonalBase<OtherDerived> &other) const + { return Product<Derived,OtherDerived>(derived(), other.derived()); } + + // diagonal * sparse + template<typename OtherDerived> friend + const Product<OtherDerived,Derived> + operator*(const DiagonalBase<OtherDerived> &lhs, const SparseMatrixBase& rhs) + { return Product<OtherDerived,Derived>(lhs.derived(), rhs.derived()); } + + // sparse * sparse + template<typename OtherDerived> + const Product<Derived,OtherDerived> + operator*(const SparseMatrixBase<OtherDerived> &other) const; + + // sparse * dense + template<typename OtherDerived> + const Product<Derived,OtherDerived> + operator*(const MatrixBase<OtherDerived> &other) const + { return Product<Derived,OtherDerived>(derived(), other.derived()); } + + // dense * sparse + template<typename OtherDerived> friend + const Product<OtherDerived,Derived> + operator*(const MatrixBase<OtherDerived> &lhs, const SparseMatrixBase& rhs) + { return Product<OtherDerived,Derived>(lhs.derived(), rhs.derived()); } +#endif // EIGEN_TEST_EVALUATORS + /** \returns an expression of P H P^-1 where H is the matrix represented by \c *this */ SparseSymmetricPermutationProduct<Derived,Upper|Lower> twistedBy(const PermutationMatrix<Dynamic,Dynamic,Index>& perm) const { @@ -371,7 +335,7 @@ template<typename Derived> class SparseMatrixBase : public EigenBase<Derived> Derived& operator*=(const SparseMatrixBase<OtherDerived>& other); template<int Mode> - inline const SparseTriangularView<Derived, Mode> triangularView() const; + inline const TriangularView<Derived, Mode> triangularView() const; template<unsigned int UpLo> inline const SparseSelfAdjointView<Derived, UpLo> selfadjointView() const; template<unsigned int UpLo> inline SparseSelfAdjointView<Derived, UpLo> selfadjointView(); @@ -396,6 +360,7 @@ template<typename Derived> class SparseMatrixBase : public EigenBase<Derived> Block<Derived,Dynamic,Dynamic,true> innerVectors(Index outerStart, Index outerSize); const Block<const Derived,Dynamic,Dynamic,true> innerVectors(Index outerStart, Index outerSize) const; +#ifndef EIGEN_TEST_EVALUATORS /** \internal use operator= */ template<typename DenseDerived> void evalTo(MatrixBase<DenseDerived>& dst) const @@ -405,6 +370,7 @@ template<typename Derived> class SparseMatrixBase : public EigenBase<Derived> for (typename Derived::InnerIterator i(derived(),typename Derived::Index(j)); i; ++i) dst.coeffRef(i.row(),i.col()) = i.value(); } +#endif Matrix<Scalar,RowsAtCompileTime,ColsAtCompileTime> toDense() const { @@ -430,6 +396,9 @@ template<typename Derived> class SparseMatrixBase : public EigenBase<Derived> { return typename internal::eval<Derived>::type(derived()); } Scalar sum() const; + + inline const SparseView<Derived> + pruned(const Scalar& reference = Scalar(0), const RealScalar& epsilon = NumTraits<Scalar>::dummy_precision()) const; protected: diff --git a/Eigen/src/SparseCore/SparsePermutation.h b/Eigen/src/SparseCore/SparsePermutation.h index b85be93f6..a888ae9e1 100644 --- a/Eigen/src/SparseCore/SparsePermutation.h +++ b/Eigen/src/SparseCore/SparsePermutation.h @@ -103,7 +103,7 @@ struct permut_sparsematrix_product_retval } - +#ifndef EIGEN_TEST_EVALUATORS /** \returns the matrix with the permutation applied to the columns */ @@ -143,6 +143,139 @@ operator*(const Transpose<PermutationBase<PermDerived> >& tperm, const SparseMat return internal::permut_sparsematrix_product_retval<PermutationBase<PermDerived>, SparseDerived, OnTheLeft, true>(tperm.nestedPermutation(), matrix.derived()); } +#else // EIGEN_TEST_EVALUATORS + +namespace internal { + +template <int ProductTag> struct product_promote_storage_type<Sparse, PermutationStorage, ProductTag> { typedef Sparse ret; }; +template <int ProductTag> struct product_promote_storage_type<PermutationStorage, Sparse, ProductTag> { typedef Sparse ret; }; + +// TODO, the following need cleaning, this is just a copy-past of the dense case + +template<typename Lhs, typename Rhs, int ProductTag> +struct generic_product_impl<Lhs, Rhs, PermutationShape, SparseShape, ProductTag> +{ + template<typename Dest> + static void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) + { + permut_sparsematrix_product_retval<Lhs, Rhs, OnTheLeft, false> pmpr(lhs, rhs); + pmpr.evalTo(dst); + } +}; + +template<typename Lhs, typename Rhs, int ProductTag> +struct generic_product_impl<Lhs, Rhs, SparseShape, PermutationShape, ProductTag> +{ + template<typename Dest> + static void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) + { + permut_sparsematrix_product_retval<Rhs, Lhs, OnTheRight, false> pmpr(rhs, lhs); + pmpr.evalTo(dst); + } +}; + +template<typename Lhs, typename Rhs, int ProductTag> +struct generic_product_impl<Transpose<Lhs>, Rhs, PermutationShape, SparseShape, ProductTag> +{ + template<typename Dest> + static void evalTo(Dest& dst, const Transpose<Lhs>& lhs, const Rhs& rhs) + { + permut_sparsematrix_product_retval<Lhs, Rhs, OnTheLeft, true> pmpr(lhs.nestedPermutation(), rhs); + pmpr.evalTo(dst); + } +}; + +template<typename Lhs, typename Rhs, int ProductTag> +struct generic_product_impl<Lhs, Transpose<Rhs>, SparseShape, PermutationShape, ProductTag> +{ + template<typename Dest> + static void evalTo(Dest& dst, const Lhs& lhs, const Transpose<Rhs>& rhs) + { + permut_sparsematrix_product_retval<Rhs, Lhs, OnTheRight, true> pmpr(rhs.nestedPermutation(), lhs); + pmpr.evalTo(dst); + } +}; + +// TODO, the following two overloads are only needed to define the right temporary type through +// typename traits<permut_sparsematrix_product_retval<Rhs,Lhs,OnTheRight,false> >::ReturnType +// while it should be correctly handled by traits<Product<> >::PlainObject + +template<typename Lhs, typename Rhs, int ProductTag> +struct product_evaluator<Product<Lhs, Rhs, DefaultProduct>, ProductTag, PermutationShape, SparseShape, typename traits<Lhs>::Scalar, typename traits<Rhs>::Scalar> + : public evaluator<typename traits<permut_sparsematrix_product_retval<Lhs,Rhs,OnTheRight,false> >::ReturnType>::type +{ + typedef Product<Lhs, Rhs, DefaultProduct> XprType; + typedef typename traits<permut_sparsematrix_product_retval<Lhs,Rhs,OnTheRight,false> >::ReturnType PlainObject; + typedef typename evaluator<PlainObject>::type Base; + + product_evaluator(const XprType& xpr) + : m_result(xpr.rows(), xpr.cols()) + { + ::new (static_cast<Base*>(this)) Base(m_result); + generic_product_impl<Lhs, Rhs, PermutationShape, SparseShape, ProductTag>::evalTo(m_result, xpr.lhs(), xpr.rhs()); + } + +protected: + PlainObject m_result; +}; + +template<typename Lhs, typename Rhs, int ProductTag> +struct product_evaluator<Product<Lhs, Rhs, DefaultProduct>, ProductTag, SparseShape, PermutationShape, typename traits<Lhs>::Scalar, typename traits<Rhs>::Scalar> + : public evaluator<typename traits<permut_sparsematrix_product_retval<Rhs,Lhs,OnTheRight,false> >::ReturnType>::type +{ + typedef Product<Lhs, Rhs, DefaultProduct> XprType; + typedef typename traits<permut_sparsematrix_product_retval<Rhs,Lhs,OnTheRight,false> >::ReturnType PlainObject; + typedef typename evaluator<PlainObject>::type Base; + + product_evaluator(const XprType& xpr) + : m_result(xpr.rows(), xpr.cols()) + { + ::new (static_cast<Base*>(this)) Base(m_result); + generic_product_impl<Lhs, Rhs, SparseShape, PermutationShape, ProductTag>::evalTo(m_result, xpr.lhs(), xpr.rhs()); + } + +protected: + PlainObject m_result; +}; + +} // end namespace internal + +/** \returns the matrix with the permutation applied to the columns + */ +template<typename SparseDerived, typename PermDerived> +inline const Product<SparseDerived, PermDerived> +operator*(const SparseMatrixBase<SparseDerived>& matrix, const PermutationBase<PermDerived>& perm) +{ return Product<SparseDerived, PermDerived>(matrix.derived(), perm.derived()); } + +/** \returns the matrix with the permutation applied to the rows + */ +template<typename SparseDerived, typename PermDerived> +inline const Product<PermDerived, SparseDerived> +operator*( const PermutationBase<PermDerived>& perm, const SparseMatrixBase<SparseDerived>& matrix) +{ return Product<PermDerived, SparseDerived>(perm.derived(), matrix.derived()); } + + +// TODO, the following specializations should not be needed as Transpose<Permutation*> should be a PermutationBase. +/** \returns the matrix with the inverse permutation applied to the columns. + */ +template<typename SparseDerived, typename PermDerived> +inline const Product<SparseDerived, Transpose<PermutationBase<PermDerived> > > +operator*(const SparseMatrixBase<SparseDerived>& matrix, const Transpose<PermutationBase<PermDerived> >& tperm) +{ + return Product<SparseDerived, Transpose<PermutationBase<PermDerived> > >(matrix.derived(), tperm); +} + +/** \returns the matrix with the inverse permutation applied to the rows. + */ +template<typename SparseDerived, typename PermDerived> +inline const Product<Transpose<PermutationBase<PermDerived> >, SparseDerived> +operator*(const Transpose<PermutationBase<PermDerived> >& tperm, const SparseMatrixBase<SparseDerived>& matrix) +{ + return Product<Transpose<PermutationBase<PermDerived> >, SparseDerived>(tperm, matrix.derived()); +} + +#endif // EIGEN_TEST_EVALUATORS + } // end namespace Eigen #endif // EIGEN_SPARSE_SELFADJOINTVIEW_H diff --git a/Eigen/src/SparseCore/SparseProduct.h b/Eigen/src/SparseCore/SparseProduct.h index cf7663070..18f40b9d9 100644 --- a/Eigen/src/SparseCore/SparseProduct.h +++ b/Eigen/src/SparseCore/SparseProduct.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2010 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -12,6 +12,8 @@ namespace Eigen { +#ifndef EIGEN_TEST_EVALUATORS + template<typename Lhs, typename Rhs> struct SparseSparseProductReturnType { @@ -183,6 +185,79 @@ SparseMatrixBase<Derived>::operator*(const SparseMatrixBase<OtherDerived> &other return typename SparseSparseProductReturnType<Derived,OtherDerived>::Type(derived(), other.derived()); } +#else // EIGEN_TEST_EVALUATORS + + +/** \returns an expression of the product of two sparse matrices. + * By default a conservative product preserving the symbolic non zeros is performed. + * The automatic pruning of the small values can be achieved by calling the pruned() function + * in which case a totally different product algorithm is employed: + * \code + * C = (A*B).pruned(); // supress numerical zeros (exact) + * C = (A*B).pruned(ref); + * C = (A*B).pruned(ref,epsilon); + * \endcode + * where \c ref is a meaningful non zero reference value. + * */ +template<typename Derived> +template<typename OtherDerived> +inline const Product<Derived,OtherDerived> +SparseMatrixBase<Derived>::operator*(const SparseMatrixBase<OtherDerived> &other) const +{ + return Product<Derived,OtherDerived>(derived(), other.derived()); +} + +namespace internal { + +template<typename Lhs, typename Rhs, int ProductType> +struct generic_product_impl<Lhs, Rhs, SparseShape, SparseShape, ProductType> +{ + template<typename Dest> + static void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) + { + typedef typename nested_eval<Lhs,Dynamic>::type LhsNested; + typedef typename nested_eval<Rhs,Dynamic>::type RhsNested; + LhsNested lhsNested(lhs); + RhsNested rhsNested(rhs); + internal::conservative_sparse_sparse_product_selector<typename remove_all<LhsNested>::type, + typename remove_all<RhsNested>::type, Dest>::run(lhsNested,rhsNested,dst); + } +}; + +template<typename Lhs, typename Rhs, int Options> +struct evaluator<SparseView<Product<Lhs, Rhs, Options> > > + : public evaluator<typename Product<Lhs, Rhs, DefaultProduct>::PlainObject>::type +{ + typedef SparseView<Product<Lhs, Rhs, Options> > XprType; + typedef typename XprType::PlainObject PlainObject; + typedef typename evaluator<PlainObject>::type Base; + + typedef evaluator type; + typedef evaluator nestedType; + + evaluator(const XprType& xpr) + : m_result(xpr.rows(), xpr.cols()) + { + using std::abs; + ::new (static_cast<Base*>(this)) Base(m_result); + typedef typename nested_eval<Lhs,Dynamic>::type LhsNested; + typedef typename nested_eval<Rhs,Dynamic>::type RhsNested; + LhsNested lhsNested(xpr.nestedExpression().lhs()); + RhsNested rhsNested(xpr.nestedExpression().rhs()); + + internal::sparse_sparse_product_with_pruning_selector<typename remove_all<LhsNested>::type, + typename remove_all<RhsNested>::type, PlainObject>::run(lhsNested,rhsNested,m_result, + abs(xpr.reference())*xpr.epsilon()); + } + +protected: + PlainObject m_result; +}; + +} // end namespace internal + +#endif // EIGEN_TEST_EVALUATORS + } // end namespace Eigen #endif // EIGEN_SPARSEPRODUCT_H diff --git a/Eigen/src/SparseCore/SparseRedux.h b/Eigen/src/SparseCore/SparseRedux.h index f3da93a71..cf78d0e91 100644 --- a/Eigen/src/SparseCore/SparseRedux.h +++ b/Eigen/src/SparseCore/SparseRedux.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -18,8 +18,14 @@ SparseMatrixBase<Derived>::sum() const { eigen_assert(rows()>0 && cols()>0 && "you are using a non initialized matrix"); Scalar res(0); +#ifndef EIGEN_TEST_EVALUATORS for (Index j=0; j<outerSize(); ++j) for (typename Derived::InnerIterator iter(derived(),j); iter; ++iter) +#else + typename internal::evaluator<Derived>::type thisEval(derived()); + for (Index j=0; j<outerSize(); ++j) + for (typename internal::evaluator<Derived>::InnerIterator iter(thisEval,j); iter; ++iter) +#endif res += iter.value(); return res; } diff --git a/Eigen/src/SparseCore/SparseSelfAdjointView.h b/Eigen/src/SparseCore/SparseSelfAdjointView.h index 56c922929..4235d6c4c 100644 --- a/Eigen/src/SparseCore/SparseSelfAdjointView.h +++ b/Eigen/src/SparseCore/SparseSelfAdjointView.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2009-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -12,13 +12,23 @@ namespace Eigen { +#ifndef EIGEN_TEST_EVALUATORS + +template<typename Lhs, typename Rhs, int Mode> +class SparseSelfAdjointTimeDenseProduct; + +template<typename Lhs, typename Rhs, int Mode> +class DenseTimeSparseSelfAdjointProduct; + +#endif // EIGEN_TEST_EVALUATORS + /** \ingroup SparseCore_Module * \class SparseSelfAdjointView * * \brief Pseudo expression to manipulate a triangular sparse matrix as a selfadjoint matrix. * * \param MatrixType the type of the dense matrix storing the coefficients - * \param UpLo can be either \c #Lower or \c #Upper + * \param Mode can be either \c #Lower or \c #Upper * * This class is an expression of a sefladjoint matrix from a triangular part of a matrix * with given dense storage of the coefficients. It is the return type of MatrixBase::selfadjointView() @@ -26,37 +36,33 @@ namespace Eigen { * * \sa SparseMatrixBase::selfadjointView() */ -template<typename Lhs, typename Rhs, int UpLo> -class SparseSelfAdjointTimeDenseProduct; - -template<typename Lhs, typename Rhs, int UpLo> -class DenseTimeSparseSelfAdjointProduct; - namespace internal { -template<typename MatrixType, unsigned int UpLo> -struct traits<SparseSelfAdjointView<MatrixType,UpLo> > : traits<MatrixType> { +template<typename MatrixType, unsigned int Mode> +struct traits<SparseSelfAdjointView<MatrixType,Mode> > : traits<MatrixType> { }; -template<int SrcUpLo,int DstUpLo,typename MatrixType,int DestOrder> +template<int SrcMode,int DstMode,typename MatrixType,int DestOrder> void permute_symm_to_symm(const MatrixType& mat, SparseMatrix<typename MatrixType::Scalar,DestOrder,typename MatrixType::Index>& _dest, const typename MatrixType::Index* perm = 0); -template<int UpLo,typename MatrixType,int DestOrder> +template<int Mode,typename MatrixType,int DestOrder> void permute_symm_to_fullsymm(const MatrixType& mat, SparseMatrix<typename MatrixType::Scalar,DestOrder,typename MatrixType::Index>& _dest, const typename MatrixType::Index* perm = 0); } -template<typename MatrixType, unsigned int UpLo> class SparseSelfAdjointView - : public EigenBase<SparseSelfAdjointView<MatrixType,UpLo> > +template<typename MatrixType, unsigned int _Mode> class SparseSelfAdjointView + : public EigenBase<SparseSelfAdjointView<MatrixType,_Mode> > { public: + + enum { Mode = _Mode }; typedef typename MatrixType::Scalar Scalar; typedef typename MatrixType::Index Index; typedef Matrix<Index,Dynamic,1> VectorI; typedef typename MatrixType::Nested MatrixTypeNested; typedef typename internal::remove_all<MatrixTypeNested>::type _MatrixTypeNested; - + inline SparseSelfAdjointView(const MatrixType& matrix) : m_matrix(matrix) { eigen_assert(rows()==cols() && "SelfAdjointView is only for squared matrices"); @@ -74,40 +80,76 @@ template<typename MatrixType, unsigned int UpLo> class SparseSelfAdjointView * Note that there is no algorithmic advantage of performing such a product compared to a general sparse-sparse matrix product. * Indeed, the SparseSelfadjointView operand is first copied into a temporary SparseMatrix before computing the product. */ +#ifndef EIGEN_TEST_EVALUATORS template<typename OtherDerived> SparseSparseProduct<typename OtherDerived::PlainObject, OtherDerived> operator*(const SparseMatrixBase<OtherDerived>& rhs) const { return SparseSparseProduct<typename OtherDerived::PlainObject, OtherDerived>(*this, rhs.derived()); } +#else + template<typename OtherDerived> + Product<SparseSelfAdjointView, OtherDerived> + operator*(const SparseMatrixBase<OtherDerived>& rhs) const + { + return Product<SparseSelfAdjointView, OtherDerived>(*this, rhs.derived()); + } +#endif /** \returns an expression of the matrix product between a sparse matrix \a lhs and a sparse self-adjoint matrix \a rhs. * * Note that there is no algorithmic advantage of performing such a product compared to a general sparse-sparse matrix product. * Indeed, the SparseSelfadjointView operand is first copied into a temporary SparseMatrix before computing the product. */ +#ifndef EIGEN_TEST_EVALUATORS template<typename OtherDerived> friend SparseSparseProduct<OtherDerived, typename OtherDerived::PlainObject > operator*(const SparseMatrixBase<OtherDerived>& lhs, const SparseSelfAdjointView& rhs) { return SparseSparseProduct<OtherDerived, typename OtherDerived::PlainObject>(lhs.derived(), rhs); } +#else // EIGEN_TEST_EVALUATORS + template<typename OtherDerived> friend + Product<OtherDerived, SparseSelfAdjointView> + operator*(const SparseMatrixBase<OtherDerived>& lhs, const SparseSelfAdjointView& rhs) + { + return Product<OtherDerived, SparseSelfAdjointView>(lhs.derived(), rhs); + } +#endif // EIGEN_TEST_EVALUATORS /** Efficient sparse self-adjoint matrix times dense vector/matrix product */ +#ifndef EIGEN_TEST_EVALUATORS + template<typename OtherDerived> + SparseSelfAdjointTimeDenseProduct<MatrixType,OtherDerived,Mode> + operator*(const MatrixBase<OtherDerived>& rhs) const + { + return SparseSelfAdjointTimeDenseProduct<MatrixType,OtherDerived,Mode>(m_matrix, rhs.derived()); + } +#else template<typename OtherDerived> - SparseSelfAdjointTimeDenseProduct<MatrixType,OtherDerived,UpLo> + Product<SparseSelfAdjointView,OtherDerived> operator*(const MatrixBase<OtherDerived>& rhs) const { - return SparseSelfAdjointTimeDenseProduct<MatrixType,OtherDerived,UpLo>(m_matrix, rhs.derived()); + return Product<SparseSelfAdjointView,OtherDerived>(*this, rhs.derived()); } +#endif /** Efficient dense vector/matrix times sparse self-adjoint matrix product */ +#ifndef EIGEN_TEST_EVALUATORS template<typename OtherDerived> friend - DenseTimeSparseSelfAdjointProduct<OtherDerived,MatrixType,UpLo> + DenseTimeSparseSelfAdjointProduct<OtherDerived,MatrixType,Mode> operator*(const MatrixBase<OtherDerived>& lhs, const SparseSelfAdjointView& rhs) { - return DenseTimeSparseSelfAdjointProduct<OtherDerived,_MatrixTypeNested,UpLo>(lhs.derived(), rhs.m_matrix); + return DenseTimeSparseSelfAdjointProduct<OtherDerived,_MatrixTypeNested,Mode>(lhs.derived(), rhs.m_matrix); } +#else + template<typename OtherDerived> friend + Product<OtherDerived,SparseSelfAdjointView> + operator*(const MatrixBase<OtherDerived>& lhs, const SparseSelfAdjointView& rhs) + { + return Product<OtherDerived,SparseSelfAdjointView>(lhs.derived(), rhs); + } +#endif /** Perform a symmetric rank K update of the selfadjoint matrix \c *this: * \f$ this = this + \alpha ( u u^* ) \f$ where \a u is a vector or matrix. @@ -123,30 +165,31 @@ template<typename MatrixType, unsigned int UpLo> class SparseSelfAdjointView /** \internal triggered by sparse_matrix = SparseSelfadjointView; */ template<typename DestScalar,int StorageOrder> void evalTo(SparseMatrix<DestScalar,StorageOrder,Index>& _dest) const { - internal::permute_symm_to_fullsymm<UpLo>(m_matrix, _dest); + internal::permute_symm_to_fullsymm<Mode>(m_matrix, _dest); } template<typename DestScalar> void evalTo(DynamicSparseMatrix<DestScalar,ColMajor,Index>& _dest) const { // TODO directly evaluate into _dest; SparseMatrix<DestScalar,ColMajor,Index> tmp(_dest.rows(),_dest.cols()); - internal::permute_symm_to_fullsymm<UpLo>(m_matrix, tmp); + internal::permute_symm_to_fullsymm<Mode>(m_matrix, tmp); _dest = tmp; } /** \returns an expression of P H P^-1 */ - SparseSymmetricPermutationProduct<_MatrixTypeNested,UpLo> twistedBy(const PermutationMatrix<Dynamic,Dynamic,Index>& perm) const +// #ifndef EIGEN_TEST_EVALUATORS + SparseSymmetricPermutationProduct<_MatrixTypeNested,Mode> twistedBy(const PermutationMatrix<Dynamic,Dynamic,Index>& perm) const { - return SparseSymmetricPermutationProduct<_MatrixTypeNested,UpLo>(m_matrix, perm); + return SparseSymmetricPermutationProduct<_MatrixTypeNested,Mode>(m_matrix, perm); } - - template<typename SrcMatrixType,int SrcUpLo> - SparseSelfAdjointView& operator=(const SparseSymmetricPermutationProduct<SrcMatrixType,SrcUpLo>& permutedMatrix) + + template<typename SrcMatrixType,int SrcMode> + SparseSelfAdjointView& operator=(const SparseSymmetricPermutationProduct<SrcMatrixType,SrcMode>& permutedMatrix) { permutedMatrix.evalTo(*this); return *this; } - +// #endif // EIGEN_TEST_EVALUATORS SparseSelfAdjointView& operator=(const SparseSelfAdjointView& src) { @@ -154,22 +197,18 @@ template<typename MatrixType, unsigned int UpLo> class SparseSelfAdjointView return *this = src.twistedBy(pnull); } - template<typename SrcMatrixType,unsigned int SrcUpLo> - SparseSelfAdjointView& operator=(const SparseSelfAdjointView<SrcMatrixType,SrcUpLo>& src) + template<typename SrcMatrixType,unsigned int SrcMode> + SparseSelfAdjointView& operator=(const SparseSelfAdjointView<SrcMatrixType,SrcMode>& src) { PermutationMatrix<Dynamic> pnull; return *this = src.twistedBy(pnull); } - - // const SparseLLT<PlainObject, UpLo> llt() const; - // const SparseLDLT<PlainObject, UpLo> ldlt() const; - protected: typename MatrixType::Nested m_matrix; - mutable VectorI m_countPerRow; - mutable VectorI m_countPerCol; + //mutable VectorI m_countPerRow; + //mutable VectorI m_countPerCol; }; /*************************************************************************** @@ -177,15 +216,15 @@ template<typename MatrixType, unsigned int UpLo> class SparseSelfAdjointView ***************************************************************************/ template<typename Derived> -template<unsigned int UpLo> -const SparseSelfAdjointView<Derived, UpLo> SparseMatrixBase<Derived>::selfadjointView() const +template<unsigned int Mode> +const SparseSelfAdjointView<Derived, Mode> SparseMatrixBase<Derived>::selfadjointView() const { return derived(); } template<typename Derived> -template<unsigned int UpLo> -SparseSelfAdjointView<Derived, UpLo> SparseMatrixBase<Derived>::selfadjointView() +template<unsigned int Mode> +SparseSelfAdjointView<Derived, Mode> SparseMatrixBase<Derived>::selfadjointView() { return derived(); } @@ -194,16 +233,16 @@ SparseSelfAdjointView<Derived, UpLo> SparseMatrixBase<Derived>::selfadjointView( * Implementation of SparseSelfAdjointView methods ***************************************************************************/ -template<typename MatrixType, unsigned int UpLo> +template<typename MatrixType, unsigned int Mode> template<typename DerivedU> -SparseSelfAdjointView<MatrixType,UpLo>& -SparseSelfAdjointView<MatrixType,UpLo>::rankUpdate(const SparseMatrixBase<DerivedU>& u, const Scalar& alpha) +SparseSelfAdjointView<MatrixType,Mode>& +SparseSelfAdjointView<MatrixType,Mode>::rankUpdate(const SparseMatrixBase<DerivedU>& u, const Scalar& alpha) { - SparseMatrix<Scalar,MatrixType::Flags&RowMajorBit?RowMajor:ColMajor> tmp = u * u.adjoint(); + SparseMatrix<Scalar,(MatrixType::Flags&RowMajorBit)?RowMajor:ColMajor> tmp = u * u.adjoint(); if(alpha==Scalar(0)) - m_matrix.const_cast_derived() = tmp.template triangularView<UpLo>(); + m_matrix.const_cast_derived() = tmp.template triangularView<Mode>(); else - m_matrix.const_cast_derived() += alpha * tmp.template triangularView<UpLo>(); + m_matrix.const_cast_derived() += alpha * tmp.template triangularView<Mode>(); return *this; } @@ -212,18 +251,19 @@ SparseSelfAdjointView<MatrixType,UpLo>::rankUpdate(const SparseMatrixBase<Derive * Implementation of sparse self-adjoint time dense matrix ***************************************************************************/ +#ifndef EIGEN_TEST_EVALUATORS namespace internal { -template<typename Lhs, typename Rhs, int UpLo> -struct traits<SparseSelfAdjointTimeDenseProduct<Lhs,Rhs,UpLo> > - : traits<ProductBase<SparseSelfAdjointTimeDenseProduct<Lhs,Rhs,UpLo>, Lhs, Rhs> > +template<typename Lhs, typename Rhs, int Mode> +struct traits<SparseSelfAdjointTimeDenseProduct<Lhs,Rhs,Mode> > + : traits<ProductBase<SparseSelfAdjointTimeDenseProduct<Lhs,Rhs,Mode>, Lhs, Rhs> > { typedef Dense StorageKind; }; } -template<typename Lhs, typename Rhs, int UpLo> +template<typename Lhs, typename Rhs, int Mode> class SparseSelfAdjointTimeDenseProduct - : public ProductBase<SparseSelfAdjointTimeDenseProduct<Lhs,Rhs,UpLo>, Lhs, Rhs> + : public ProductBase<SparseSelfAdjointTimeDenseProduct<Lhs,Rhs,Mode>, Lhs, Rhs> { public: EIGEN_PRODUCT_PUBLIC_INTERFACE(SparseSelfAdjointTimeDenseProduct) @@ -241,9 +281,9 @@ class SparseSelfAdjointTimeDenseProduct enum { LhsIsRowMajor = (_Lhs::Flags&RowMajorBit)==RowMajorBit, ProcessFirstHalf = - ((UpLo&(Upper|Lower))==(Upper|Lower)) - || ( (UpLo&Upper) && !LhsIsRowMajor) - || ( (UpLo&Lower) && LhsIsRowMajor), + ((Mode&(Upper|Lower))==(Upper|Lower)) + || ( (Mode&Upper) && !LhsIsRowMajor) + || ( (Mode&Lower) && LhsIsRowMajor), ProcessSecondHalf = !ProcessFirstHalf }; for (typename _Lhs::Index j=0; j<m_lhs.outerSize(); ++j) @@ -276,15 +316,15 @@ class SparseSelfAdjointTimeDenseProduct }; namespace internal { -template<typename Lhs, typename Rhs, int UpLo> -struct traits<DenseTimeSparseSelfAdjointProduct<Lhs,Rhs,UpLo> > - : traits<ProductBase<DenseTimeSparseSelfAdjointProduct<Lhs,Rhs,UpLo>, Lhs, Rhs> > +template<typename Lhs, typename Rhs, int Mode> +struct traits<DenseTimeSparseSelfAdjointProduct<Lhs,Rhs,Mode> > + : traits<ProductBase<DenseTimeSparseSelfAdjointProduct<Lhs,Rhs,Mode>, Lhs, Rhs> > {}; } -template<typename Lhs, typename Rhs, int UpLo> +template<typename Lhs, typename Rhs, int Mode> class DenseTimeSparseSelfAdjointProduct - : public ProductBase<DenseTimeSparseSelfAdjointProduct<Lhs,Rhs,UpLo>, Lhs, Rhs> + : public ProductBase<DenseTimeSparseSelfAdjointProduct<Lhs,Rhs,Mode>, Lhs, Rhs> { public: EIGEN_PRODUCT_PUBLIC_INTERFACE(DenseTimeSparseSelfAdjointProduct) @@ -301,16 +341,159 @@ class DenseTimeSparseSelfAdjointProduct DenseTimeSparseSelfAdjointProduct& operator=(const DenseTimeSparseSelfAdjointProduct&); }; +#else // EIGEN_TEST_EVALUATORS + +namespace internal { + +template<int Mode, typename SparseLhsType, typename DenseRhsType, typename DenseResType, typename AlphaType> +inline void sparse_selfadjoint_time_dense_product(const SparseLhsType& lhs, const DenseRhsType& rhs, DenseResType& res, const AlphaType& alpha) +{ + EIGEN_ONLY_USED_FOR_DEBUG(alpha); + // TODO use alpha + eigen_assert(alpha==AlphaType(1) && "alpha != 1 is not implemented yet, sorry"); + + typedef typename evaluator<SparseLhsType>::type LhsEval; + typedef typename evaluator<SparseLhsType>::InnerIterator LhsIterator; + typedef typename SparseLhsType::Index Index; + typedef typename SparseLhsType::Scalar LhsScalar; + + enum { + LhsIsRowMajor = (LhsEval::Flags&RowMajorBit)==RowMajorBit, + ProcessFirstHalf = + ((Mode&(Upper|Lower))==(Upper|Lower)) + || ( (Mode&Upper) && !LhsIsRowMajor) + || ( (Mode&Lower) && LhsIsRowMajor), + ProcessSecondHalf = !ProcessFirstHalf + }; + + LhsEval lhsEval(lhs); + + for (Index j=0; j<lhs.outerSize(); ++j) + { + LhsIterator i(lhsEval,j); + if (ProcessSecondHalf) + { + while (i && i.index()<j) ++i; + if(i && i.index()==j) + { + res.row(j) += i.value() * rhs.row(j); + ++i; + } + } + for(; (ProcessFirstHalf ? i && i.index() < j : i) ; ++i) + { + Index a = LhsIsRowMajor ? j : i.index(); + Index b = LhsIsRowMajor ? i.index() : j; + LhsScalar v = i.value(); + res.row(a) += (v) * rhs.row(b); + res.row(b) += numext::conj(v) * rhs.row(a); + } + if (ProcessFirstHalf && i && (i.index()==j)) + res.row(j) += i.value() * rhs.row(j); + } +} + +// TODO currently a selfadjoint expression has the form SelfAdjointView<.,.> +// in the future selfadjoint-ness should be defined by the expression traits +// such that Transpose<SelfAdjointView<.,.> > is valid. (currently TriangularBase::transpose() is overloaded to make it work) +template<typename MatrixType, unsigned int Mode> +struct evaluator_traits<SparseSelfAdjointView<MatrixType,Mode> > +{ + typedef typename storage_kind_to_evaluator_kind<typename MatrixType::StorageKind>::Kind Kind; + typedef SparseSelfAdjointShape Shape; + + static const int AssumeAliasing = 0; +}; + +template<typename LhsView, typename Rhs, int ProductType> +struct generic_product_impl<LhsView, Rhs, SparseSelfAdjointShape, DenseShape, ProductType> +{ + template<typename Dest> + static void evalTo(Dest& dst, const LhsView& lhsView, const Rhs& rhs) + { + typedef typename LhsView::_MatrixTypeNested Lhs; + typedef typename nested_eval<Lhs,Dynamic>::type LhsNested; + typedef typename nested_eval<Rhs,Dynamic>::type RhsNested; + LhsNested lhsNested(lhsView.matrix()); + RhsNested rhsNested(rhs); + + dst.setZero(); + internal::sparse_selfadjoint_time_dense_product<LhsView::Mode>(lhsNested, rhsNested, dst, typename Dest::Scalar(1)); + } +}; + +template<typename Lhs, typename RhsView, int ProductType> +struct generic_product_impl<Lhs, RhsView, DenseShape, SparseSelfAdjointShape, ProductType> +{ + template<typename Dest> + static void evalTo(Dest& dst, const Lhs& lhs, const RhsView& rhsView) + { + typedef typename RhsView::_MatrixTypeNested Rhs; + typedef typename nested_eval<Lhs,Dynamic>::type LhsNested; + typedef typename nested_eval<Rhs,Dynamic>::type RhsNested; + LhsNested lhsNested(lhs); + RhsNested rhsNested(rhsView.matrix()); + + dst.setZero(); + // transpoe everything + Transpose<Dest> dstT(dst); + internal::sparse_selfadjoint_time_dense_product<RhsView::Mode>(rhsNested.transpose(), lhsNested.transpose(), dstT, typename Dest::Scalar(1)); + } +}; + +// NOTE: these two overloads are needed to evaluate the sparse sefladjoint view into a full sparse matrix +// TODO: maybe the copy could be handled by generic_product_impl so that these overloads would not be needed anymore + +template<typename LhsView, typename Rhs, int ProductTag> +struct product_evaluator<Product<LhsView, Rhs, DefaultProduct>, ProductTag, SparseSelfAdjointShape, SparseShape, typename traits<LhsView>::Scalar, typename traits<Rhs>::Scalar> + : public evaluator<typename Product<typename Rhs::PlainObject, Rhs, DefaultProduct>::PlainObject>::type +{ + typedef Product<LhsView, Rhs, DefaultProduct> XprType; + typedef typename XprType::PlainObject PlainObject; + typedef typename evaluator<PlainObject>::type Base; + + product_evaluator(const XprType& xpr) + : m_lhs(xpr.lhs()), m_result(xpr.rows(), xpr.cols()) + { + ::new (static_cast<Base*>(this)) Base(m_result); + generic_product_impl<typename Rhs::PlainObject, Rhs, SparseShape, SparseShape, ProductTag>::evalTo(m_result, m_lhs, xpr.rhs()); + } + +protected: + typename Rhs::PlainObject m_lhs; + PlainObject m_result; +}; + +template<typename Lhs, typename RhsView, int ProductTag> +struct product_evaluator<Product<Lhs, RhsView, DefaultProduct>, ProductTag, SparseShape, SparseSelfAdjointShape, typename traits<Lhs>::Scalar, typename traits<RhsView>::Scalar> + : public evaluator<typename Product<Lhs, typename Lhs::PlainObject, DefaultProduct>::PlainObject>::type +{ + typedef Product<Lhs, RhsView, DefaultProduct> XprType; + typedef typename XprType::PlainObject PlainObject; + typedef typename evaluator<PlainObject>::type Base; + + product_evaluator(const XprType& xpr) + : m_rhs(xpr.rhs()), m_result(xpr.rows(), xpr.cols()) + { + ::new (static_cast<Base*>(this)) Base(m_result); + generic_product_impl<Lhs, typename Lhs::PlainObject, SparseShape, SparseShape, ProductTag>::evalTo(m_result, xpr.lhs(), m_rhs); + } + +protected: + typename Lhs::PlainObject m_rhs; + PlainObject m_result; +}; + +} // namespace internal + +#endif // EIGEN_TEST_EVALUATORS + /*************************************************************************** * Implementation of symmetric copies and permutations ***************************************************************************/ namespace internal { - -template<typename MatrixType, int UpLo> -struct traits<SparseSymmetricPermutationProduct<MatrixType,UpLo> > : traits<MatrixType> { -}; -template<int UpLo,typename MatrixType,int DestOrder> +template<int Mode,typename MatrixType,int DestOrder> void permute_symm_to_fullsymm(const MatrixType& mat, SparseMatrix<typename MatrixType::Scalar,DestOrder,typename MatrixType::Index>& _dest, const typename MatrixType::Index* perm) { typedef typename MatrixType::Index Index; @@ -337,11 +520,11 @@ void permute_symm_to_fullsymm(const MatrixType& mat, SparseMatrix<typename Matri Index r = it.row(); Index c = it.col(); Index ip = perm ? perm[i] : i; - if(UpLo==(Upper|Lower)) + if(Mode==(Upper|Lower)) count[StorageOrderMatch ? jp : ip]++; else if(r==c) count[ip]++; - else if(( UpLo==Lower && r>c) || ( UpLo==Upper && r<c)) + else if(( Mode==Lower && r>c) || ( Mode==Upper && r<c)) { count[ip]++; count[jp]++; @@ -370,7 +553,7 @@ void permute_symm_to_fullsymm(const MatrixType& mat, SparseMatrix<typename Matri Index jp = perm ? perm[j] : j; Index ip = perm ? perm[i] : i; - if(UpLo==(Upper|Lower)) + if(Mode==(Upper|Lower)) { Index k = count[StorageOrderMatch ? jp : ip]++; dest.innerIndexPtr()[k] = StorageOrderMatch ? ip : jp; @@ -382,7 +565,7 @@ void permute_symm_to_fullsymm(const MatrixType& mat, SparseMatrix<typename Matri dest.innerIndexPtr()[k] = ip; dest.valuePtr()[k] = it.value(); } - else if(( (UpLo&Lower)==Lower && r>c) || ( (UpLo&Upper)==Upper && r<c)) + else if(( (Mode&Lower)==Lower && r>c) || ( (Mode&Upper)==Upper && r<c)) { if(!StorageOrderMatch) std::swap(ip,jp); @@ -397,7 +580,7 @@ void permute_symm_to_fullsymm(const MatrixType& mat, SparseMatrix<typename Matri } } -template<int _SrcUpLo,int _DstUpLo,typename MatrixType,int DstOrder> +template<int _SrcMode,int _DstMode,typename MatrixType,int DstOrder> void permute_symm_to_symm(const MatrixType& mat, SparseMatrix<typename MatrixType::Scalar,DstOrder,typename MatrixType::Index>& _dest, const typename MatrixType::Index* perm) { typedef typename MatrixType::Index Index; @@ -407,8 +590,8 @@ void permute_symm_to_symm(const MatrixType& mat, SparseMatrix<typename MatrixTyp enum { SrcOrder = MatrixType::IsRowMajor ? RowMajor : ColMajor, StorageOrderMatch = int(SrcOrder) == int(DstOrder), - DstUpLo = DstOrder==RowMajor ? (_DstUpLo==Upper ? Lower : Upper) : _DstUpLo, - SrcUpLo = SrcOrder==RowMajor ? (_SrcUpLo==Upper ? Lower : Upper) : _SrcUpLo + DstMode = DstOrder==RowMajor ? (_DstMode==Upper ? Lower : Upper) : _DstMode, + SrcMode = SrcOrder==RowMajor ? (_SrcMode==Upper ? Lower : Upper) : _SrcMode }; Index size = mat.rows(); @@ -421,11 +604,11 @@ void permute_symm_to_symm(const MatrixType& mat, SparseMatrix<typename MatrixTyp for(typename MatrixType::InnerIterator it(mat,j); it; ++it) { Index i = it.index(); - if((int(SrcUpLo)==int(Lower) && i<j) || (int(SrcUpLo)==int(Upper) && i>j)) + if((int(SrcMode)==int(Lower) && i<j) || (int(SrcMode)==int(Upper) && i>j)) continue; Index ip = perm ? perm[i] : i; - count[int(DstUpLo)==int(Lower) ? (std::min)(ip,jp) : (std::max)(ip,jp)]++; + count[int(DstMode)==int(Lower) ? (std::min)(ip,jp) : (std::max)(ip,jp)]++; } } dest.outerIndexPtr()[0] = 0; @@ -441,17 +624,17 @@ void permute_symm_to_symm(const MatrixType& mat, SparseMatrix<typename MatrixTyp for(typename MatrixType::InnerIterator it(mat,j); it; ++it) { Index i = it.index(); - if((int(SrcUpLo)==int(Lower) && i<j) || (int(SrcUpLo)==int(Upper) && i>j)) + if((int(SrcMode)==int(Lower) && i<j) || (int(SrcMode)==int(Upper) && i>j)) continue; Index jp = perm ? perm[j] : j; Index ip = perm? perm[i] : i; - Index k = count[int(DstUpLo)==int(Lower) ? (std::min)(ip,jp) : (std::max)(ip,jp)]++; - dest.innerIndexPtr()[k] = int(DstUpLo)==int(Lower) ? (std::max)(ip,jp) : (std::min)(ip,jp); + Index k = count[int(DstMode)==int(Lower) ? (std::min)(ip,jp) : (std::max)(ip,jp)]++; + dest.innerIndexPtr()[k] = int(DstMode)==int(Lower) ? (std::max)(ip,jp) : (std::min)(ip,jp); if(!StorageOrderMatch) std::swap(ip,jp); - if( ((int(DstUpLo)==int(Lower) && ip<jp) || (int(DstUpLo)==int(Upper) && ip>jp))) + if( ((int(DstMode)==int(Lower) && ip<jp) || (int(DstMode)==int(Upper) && ip>jp))) dest.valuePtr()[k] = numext::conj(it.value()); else dest.valuePtr()[k] = it.value(); @@ -461,9 +644,19 @@ void permute_symm_to_symm(const MatrixType& mat, SparseMatrix<typename MatrixTyp } -template<typename MatrixType,int UpLo> +// #ifndef EIGEN_TEST_EVALUATORS + +namespace internal { + +template<typename MatrixType, int Mode> +struct traits<SparseSymmetricPermutationProduct<MatrixType,Mode> > : traits<MatrixType> { +}; + +} + +template<typename MatrixType,int Mode> class SparseSymmetricPermutationProduct - : public EigenBase<SparseSymmetricPermutationProduct<MatrixType,UpLo> > + : public EigenBase<SparseSymmetricPermutationProduct<MatrixType,Mode> > { public: typedef typename MatrixType::Scalar Scalar; @@ -485,15 +678,15 @@ class SparseSymmetricPermutationProduct template<typename DestScalar, int Options, typename DstIndex> void evalTo(SparseMatrix<DestScalar,Options,DstIndex>& _dest) const { -// internal::permute_symm_to_fullsymm<UpLo>(m_matrix,_dest,m_perm.indices().data()); +// internal::permute_symm_to_fullsymm<Mode>(m_matrix,_dest,m_perm.indices().data()); SparseMatrix<DestScalar,(Options&RowMajor)==RowMajor ? ColMajor : RowMajor, DstIndex> tmp; - internal::permute_symm_to_fullsymm<UpLo>(m_matrix,tmp,m_perm.indices().data()); + internal::permute_symm_to_fullsymm<Mode>(m_matrix,tmp,m_perm.indices().data()); _dest = tmp; } - template<typename DestType,unsigned int DestUpLo> void evalTo(SparseSelfAdjointView<DestType,DestUpLo>& dest) const + template<typename DestType,unsigned int DestMode> void evalTo(SparseSelfAdjointView<DestType,DestMode>& dest) const { - internal::permute_symm_to_symm<UpLo,DestUpLo>(m_matrix,dest.matrix(),m_perm.indices().data()); + internal::permute_symm_to_symm<Mode,DestMode>(m_matrix,dest.matrix(),m_perm.indices().data()); } protected: @@ -502,6 +695,10 @@ class SparseSymmetricPermutationProduct }; +// #else // EIGEN_TEST_EVALUATORS + +// #endif // EIGEN_TEST_EVALUATORS + } // end namespace Eigen #endif // EIGEN_SPARSE_SELFADJOINTVIEW_H diff --git a/Eigen/src/SparseCore/SparseSparseProductWithPruning.h b/Eigen/src/SparseCore/SparseSparseProductWithPruning.h index fcc18f5c9..b42b33e55 100644 --- a/Eigen/src/SparseCore/SparseSparseProductWithPruning.h +++ b/Eigen/src/SparseCore/SparseSparseProductWithPruning.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2011 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -46,6 +46,11 @@ static void sparse_sparse_product_with_pruning_impl(const Lhs& lhs, const Rhs& r res.resize(cols, rows); else res.resize(rows, cols); + + #ifdef EIGEN_TEST_EVALUATORS + typename evaluator<Lhs>::type lhsEval(lhs); + typename evaluator<Rhs>::type rhsEval(rhs); + #endif res.reserve(estimated_nnz_prod); double ratioColRes = double(estimated_nnz_prod)/double(lhs.rows()*rhs.cols()); @@ -56,12 +61,20 @@ static void sparse_sparse_product_with_pruning_impl(const Lhs& lhs, const Rhs& r // let's do a more accurate determination of the nnz ratio for the current column j of res tempVector.init(ratioColRes); tempVector.setZero(); +#ifndef EIGEN_TEST_EVALUATORS for (typename Rhs::InnerIterator rhsIt(rhs, j); rhsIt; ++rhsIt) +#else + for (typename evaluator<Rhs>::InnerIterator rhsIt(rhsEval, j); rhsIt; ++rhsIt) +#endif { // FIXME should be written like this: tmp += rhsIt.value() * lhs.col(rhsIt.index()) tempVector.restart(); Scalar x = rhsIt.value(); +#ifndef EIGEN_TEST_EVALUATORS for (typename Lhs::InnerIterator lhsIt(lhs, rhsIt.index()); lhsIt; ++lhsIt) +#else + for (typename evaluator<Lhs>::InnerIterator lhsIt(lhsEval, rhsIt.index()); lhsIt; ++lhsIt) +#endif { tempVector.coeffRef(lhsIt.index()) += lhsIt.value() * x; } @@ -140,8 +153,58 @@ struct sparse_sparse_product_with_pruning_selector<Lhs,Rhs,ResultType,RowMajor,R } }; +#ifndef EIGEN_TEST_EVALUATORS // NOTE the 2 others cases (col row *) must never occur since they are caught // by ProductReturnType which transforms it to (col col *) by evaluating rhs. +#else +template<typename Lhs, typename Rhs, typename ResultType> +struct sparse_sparse_product_with_pruning_selector<Lhs,Rhs,ResultType,ColMajor,RowMajor,RowMajor> +{ + typedef typename ResultType::RealScalar RealScalar; + static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res, const RealScalar& tolerance) + { + typedef SparseMatrix<typename ResultType::Scalar,RowMajor,typename Lhs::Index> RowMajorMatrixLhs; + RowMajorMatrixLhs rowLhs(lhs); + sparse_sparse_product_with_pruning_selector<RowMajorMatrixLhs,Rhs,ResultType,RowMajor,RowMajor>(rowLhs,rhs,res,tolerance); + } +}; + +template<typename Lhs, typename Rhs, typename ResultType> +struct sparse_sparse_product_with_pruning_selector<Lhs,Rhs,ResultType,RowMajor,ColMajor,RowMajor> +{ + typedef typename ResultType::RealScalar RealScalar; + static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res, const RealScalar& tolerance) + { + typedef SparseMatrix<typename ResultType::Scalar,RowMajor,typename Lhs::Index> RowMajorMatrixRhs; + RowMajorMatrixRhs rowRhs(rhs); + sparse_sparse_product_with_pruning_selector<Lhs,RowMajorMatrixRhs,ResultType,RowMajor,RowMajor,RowMajor>(lhs,rowRhs,res,tolerance); + } +}; + +template<typename Lhs, typename Rhs, typename ResultType> +struct sparse_sparse_product_with_pruning_selector<Lhs,Rhs,ResultType,ColMajor,RowMajor,ColMajor> +{ + typedef typename ResultType::RealScalar RealScalar; + static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res, const RealScalar& tolerance) + { + typedef SparseMatrix<typename ResultType::Scalar,ColMajor,typename Lhs::Index> ColMajorMatrixRhs; + ColMajorMatrixRhs colRhs(rhs); + internal::sparse_sparse_product_with_pruning_impl<Lhs,ColMajorMatrixRhs,ResultType>(lhs, colRhs, res, tolerance); + } +}; + +template<typename Lhs, typename Rhs, typename ResultType> +struct sparse_sparse_product_with_pruning_selector<Lhs,Rhs,ResultType,RowMajor,ColMajor,ColMajor> +{ + typedef typename ResultType::RealScalar RealScalar; + static void run(const Lhs& lhs, const Rhs& rhs, ResultType& res, const RealScalar& tolerance) + { + typedef SparseMatrix<typename ResultType::Scalar,ColMajor,typename Lhs::Index> ColMajorMatrixLhs; + ColMajorMatrixLhs colLhs(lhs); + internal::sparse_sparse_product_with_pruning_impl<ColMajorMatrixLhs,Rhs,ResultType>(colLhs, rhs, res, tolerance); + } +}; +#endif } // end namespace internal diff --git a/Eigen/src/SparseCore/SparseTranspose.h b/Eigen/src/SparseCore/SparseTranspose.h index 7c300ee8d..f5eff6133 100644 --- a/Eigen/src/SparseCore/SparseTranspose.h +++ b/Eigen/src/SparseCore/SparseTranspose.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -12,6 +12,7 @@ namespace Eigen { +#ifndef EIGEN_TEST_EVALUATORS template<typename MatrixType> class TransposeImpl<MatrixType,Sparse> : public SparseMatrixBase<Transpose<MatrixType> > { @@ -58,6 +59,68 @@ template<typename MatrixType> class TransposeImpl<MatrixType,Sparse>::ReverseInn Index col() const { return Base::row(); } }; +#else // EIGEN_TEST_EVALUATORS + +// Implement nonZeros() for transpose. I'm not sure that's the best approach for that. +// Perhaps it should be implemented in Transpose<> itself. +template<typename MatrixType> class TransposeImpl<MatrixType,Sparse> + : public SparseMatrixBase<Transpose<MatrixType> > +{ + protected: + typedef SparseMatrixBase<Transpose<MatrixType> > Base; + public: + inline typename MatrixType::Index nonZeros() const { return Base::derived().nestedExpression().nonZeros(); } +}; + +namespace internal { + +template<typename ArgType> +struct unary_evaluator<Transpose<ArgType>, IteratorBased> + : public evaluator_base<Transpose<ArgType> > +{ + typedef typename evaluator<ArgType>::InnerIterator EvalIterator; + typedef typename evaluator<ArgType>::ReverseInnerIterator EvalReverseIterator; + public: + typedef Transpose<ArgType> XprType; + typedef typename XprType::Index Index; + + class InnerIterator : public EvalIterator + { + public: + EIGEN_STRONG_INLINE InnerIterator(const unary_evaluator& unaryOp, typename XprType::Index outer) + : EvalIterator(unaryOp.m_argImpl,outer) + {} + + Index row() const { return EvalIterator::col(); } + Index col() const { return EvalIterator::row(); } + }; + + class ReverseInnerIterator : public EvalReverseIterator + { + public: + EIGEN_STRONG_INLINE ReverseInnerIterator(const unary_evaluator& unaryOp, typename XprType::Index outer) + : EvalReverseIterator(unaryOp.m_argImpl,outer) + {} + + Index row() const { return EvalReverseIterator::col(); } + Index col() const { return EvalReverseIterator::row(); } + }; + + enum { + CoeffReadCost = evaluator<ArgType>::CoeffReadCost, + Flags = XprType::Flags + }; + + unary_evaluator(const XprType& op) :m_argImpl(op.nestedExpression()) {} + + protected: + typename evaluator<ArgType>::nestedType m_argImpl; +}; + +} // end namespace internal + +#endif // EIGEN_TEST_EVALUATORS + } // end namespace Eigen #endif // EIGEN_SPARSETRANSPOSE_H diff --git a/Eigen/src/SparseCore/SparseTriangularView.h b/Eigen/src/SparseCore/SparseTriangularView.h index 333127b78..87f4ab18d 100644 --- a/Eigen/src/SparseCore/SparseTriangularView.h +++ b/Eigen/src/SparseCore/SparseTriangularView.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2009-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // Copyright (C) 2012 Désiré Nuentsa-Wakam <desire.nuentsa_wakam@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla @@ -13,17 +13,8 @@ namespace Eigen { -namespace internal { - -template<typename MatrixType, int Mode> -struct traits<SparseTriangularView<MatrixType,Mode> > -: public traits<MatrixType> -{}; - -} // namespace internal - -template<typename MatrixType, int Mode> class SparseTriangularView - : public SparseMatrixBase<SparseTriangularView<MatrixType,Mode> > +template<typename MatrixType, unsigned int Mode> class TriangularViewImpl<MatrixType,Mode,Sparse> + : public SparseMatrixBase<TriangularView<MatrixType,Mode> > { enum { SkipFirst = ((Mode&Lower) && !(MatrixType::Flags&RowMajorBit)) || ((Mode&Upper) && (MatrixType::Flags&RowMajorBit)), @@ -31,46 +22,52 @@ template<typename MatrixType, int Mode> class SparseTriangularView SkipDiag = (Mode&ZeroDiag) ? 1 : 0, HasUnitDiag = (Mode&UnitDiag) ? 1 : 0 }; + + typedef TriangularView<MatrixType,Mode> TriangularViewType; + +protected: + // dummy solve function to make TriangularView happy. + void solve() const; public: - EIGEN_SPARSE_PUBLIC_INTERFACE(SparseTriangularView) - + EIGEN_SPARSE_PUBLIC_INTERFACE(TriangularViewType) + class InnerIterator; class ReverseInnerIterator; - inline Index rows() const { return m_matrix.rows(); } - inline Index cols() const { return m_matrix.cols(); } - typedef typename MatrixType::Nested MatrixTypeNested; typedef typename internal::remove_reference<MatrixTypeNested>::type MatrixTypeNestedNonRef; typedef typename internal::remove_all<MatrixTypeNested>::type MatrixTypeNestedCleaned; - inline SparseTriangularView(const MatrixType& matrix) : m_matrix(matrix) {} - - /** \internal */ - inline const MatrixTypeNestedCleaned& nestedExpression() const { return m_matrix; } - +#ifndef EIGEN_TEST_EVALUATORS template<typename OtherDerived> typename internal::plain_matrix_type_column_major<OtherDerived>::type solve(const MatrixBase<OtherDerived>& other) const; +#else // EIGEN_TEST_EVALUATORS + template<typename RhsType, typename DstType> + EIGEN_DEVICE_FUNC + EIGEN_STRONG_INLINE void _solve_impl(const RhsType &rhs, DstType &dst) const { + if(!(internal::is_same<RhsType,DstType>::value && internal::extract_data(dst) == internal::extract_data(rhs))) + dst = rhs; + this->template solveInPlace(dst); + } +#endif // EIGEN_TEST_EVALUATORS template<typename OtherDerived> void solveInPlace(MatrixBase<OtherDerived>& other) const; template<typename OtherDerived> void solveInPlace(SparseMatrixBase<OtherDerived>& other) const; - - protected: - MatrixTypeNested m_matrix; + }; -template<typename MatrixType, int Mode> -class SparseTriangularView<MatrixType,Mode>::InnerIterator : public MatrixTypeNestedCleaned::InnerIterator +template<typename MatrixType, unsigned int Mode> +class TriangularViewImpl<MatrixType,Mode,Sparse>::InnerIterator : public MatrixTypeNestedCleaned::InnerIterator { typedef typename MatrixTypeNestedCleaned::InnerIterator Base; - typedef typename SparseTriangularView::Index Index; + typedef typename TriangularViewType::Index Index; public: - EIGEN_STRONG_INLINE InnerIterator(const SparseTriangularView& view, Index outer) - : Base(view.nestedExpression(), outer), m_returnOne(false) + EIGEN_STRONG_INLINE InnerIterator(const TriangularViewImpl& view, Index outer) + : Base(view.derived().nestedExpression(), outer), m_returnOne(false) { if(SkipFirst) { @@ -132,15 +129,15 @@ class SparseTriangularView<MatrixType,Mode>::InnerIterator : public MatrixTypeNe bool m_returnOne; }; -template<typename MatrixType, int Mode> -class SparseTriangularView<MatrixType,Mode>::ReverseInnerIterator : public MatrixTypeNestedCleaned::ReverseInnerIterator +template<typename MatrixType, unsigned int Mode> +class TriangularViewImpl<MatrixType,Mode,Sparse>::ReverseInnerIterator : public MatrixTypeNestedCleaned::ReverseInnerIterator { typedef typename MatrixTypeNestedCleaned::ReverseInnerIterator Base; - typedef typename SparseTriangularView::Index Index; + typedef typename TriangularViewImpl::Index Index; public: - EIGEN_STRONG_INLINE ReverseInnerIterator(const SparseTriangularView& view, Index outer) - : Base(view.nestedExpression(), outer) + EIGEN_STRONG_INLINE ReverseInnerIterator(const TriangularViewType& view, Index outer) + : Base(view.derived().nestedExpression(), outer) { eigen_assert((!HasUnitDiag) && "ReverseInnerIterator does not support yet triangular views with a unit diagonal"); if(SkipLast) { @@ -166,9 +163,118 @@ class SparseTriangularView<MatrixType,Mode>::ReverseInnerIterator : public Matri } }; +#ifdef EIGEN_TEST_EVALUATORS +namespace internal { + +template<typename ArgType, unsigned int Mode> +struct unary_evaluator<TriangularView<ArgType,Mode>, IteratorBased> + : evaluator_base<TriangularView<ArgType,Mode> > +{ + typedef TriangularView<ArgType,Mode> XprType; + +protected: + + typedef typename XprType::Scalar Scalar; + typedef typename XprType::Index Index; + typedef typename evaluator<ArgType>::InnerIterator EvalIterator; + + enum { SkipFirst = ((Mode&Lower) && !(ArgType::Flags&RowMajorBit)) + || ((Mode&Upper) && (ArgType::Flags&RowMajorBit)), + SkipLast = !SkipFirst, + SkipDiag = (Mode&ZeroDiag) ? 1 : 0, + HasUnitDiag = (Mode&UnitDiag) ? 1 : 0 + }; + +public: + + enum { + CoeffReadCost = evaluator<ArgType>::CoeffReadCost, + Flags = XprType::Flags + }; + + unary_evaluator(const XprType &xpr) : m_argImpl(xpr.nestedExpression()) {} + + class InnerIterator : public EvalIterator + { + typedef EvalIterator Base; + public: + + EIGEN_STRONG_INLINE InnerIterator(const unary_evaluator& xprEval, Index outer) + : Base(xprEval.m_argImpl,outer), m_returnOne(false) + { + if(SkipFirst) + { + while((*this) && ((HasUnitDiag||SkipDiag) ? this->index()<=outer : this->index()<outer)) + Base::operator++(); + if(HasUnitDiag) + m_returnOne = true; + } + else if(HasUnitDiag && ((!Base::operator bool()) || Base::index()>=Base::outer())) + { + if((!SkipFirst) && Base::operator bool()) + Base::operator++(); + m_returnOne = true; + } + } + + EIGEN_STRONG_INLINE InnerIterator& operator++() + { + if(HasUnitDiag && m_returnOne) + m_returnOne = false; + else + { + Base::operator++(); + if(HasUnitDiag && (!SkipFirst) && ((!Base::operator bool()) || Base::index()>=Base::outer())) + { + if((!SkipFirst) && Base::operator bool()) + Base::operator++(); + m_returnOne = true; + } + } + return *this; + } + + EIGEN_STRONG_INLINE operator bool() const + { + if(HasUnitDiag && m_returnOne) + return true; + if(SkipFirst) return Base::operator bool(); + else + { + if (SkipDiag) return (Base::operator bool() && this->index() < this->outer()); + else return (Base::operator bool() && this->index() <= this->outer()); + } + } + +// inline Index row() const { return (ArgType::Flags&RowMajorBit ? Base::outer() : this->index()); } +// inline Index col() const { return (ArgType::Flags&RowMajorBit ? this->index() : Base::outer()); } + inline Index index() const + { + if(HasUnitDiag && m_returnOne) return Base::outer(); + else return Base::index(); + } + inline Scalar value() const + { + if(HasUnitDiag && m_returnOne) return Scalar(1); + else return Base::value(); + } + + protected: + bool m_returnOne; + private: + Scalar& valueRef(); + }; + +protected: + typename evaluator<ArgType>::type m_argImpl; +}; + +} // end namespace internal +#endif + template<typename Derived> template<int Mode> -inline const SparseTriangularView<Derived, Mode> +inline const TriangularView<Derived, Mode> SparseMatrixBase<Derived>::triangularView() const { return derived(); diff --git a/Eigen/src/SparseCore/SparseUtil.h b/Eigen/src/SparseCore/SparseUtil.h index 02c19d18f..0183907a1 100644 --- a/Eigen/src/SparseCore/SparseUtil.h +++ b/Eigen/src/SparseCore/SparseUtil.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -43,6 +43,8 @@ EIGEN_SPARSE_INHERIT_ASSIGNMENT_OPERATOR(Derived, -=) \ EIGEN_SPARSE_INHERIT_SCALAR_ASSIGNMENT_OPERATOR(Derived, *=) \ EIGEN_SPARSE_INHERIT_SCALAR_ASSIGNMENT_OPERATOR(Derived, /=) +#ifndef EIGEN_TEST_EVALUATORS + #define _EIGEN_SPARSE_PUBLIC_INTERFACE(Derived, BaseClass) \ typedef BaseClass Base; \ typedef typename Eigen::internal::traits<Derived >::Scalar Scalar; \ @@ -52,13 +54,32 @@ EIGEN_SPARSE_INHERIT_SCALAR_ASSIGNMENT_OPERATOR(Derived, /=) typedef typename Eigen::internal::traits<Derived >::Index Index; \ enum { RowsAtCompileTime = Eigen::internal::traits<Derived >::RowsAtCompileTime, \ ColsAtCompileTime = Eigen::internal::traits<Derived >::ColsAtCompileTime, \ - Flags = Eigen::internal::traits<Derived >::Flags, \ + Flags = Eigen::internal::traits<Derived>::Flags, \ CoeffReadCost = Eigen::internal::traits<Derived >::CoeffReadCost, \ SizeAtCompileTime = Base::SizeAtCompileTime, \ IsVectorAtCompileTime = Base::IsVectorAtCompileTime }; \ using Base::derived; \ using Base::const_cast_derived; +#else // EIGEN_TEST_EVALUATORS + +#define _EIGEN_SPARSE_PUBLIC_INTERFACE(Derived, BaseClass) \ + typedef BaseClass Base; \ + typedef typename Eigen::internal::traits<Derived >::Scalar Scalar; \ + typedef typename Eigen::NumTraits<Scalar>::Real RealScalar; \ + typedef typename Eigen::internal::nested<Derived >::type Nested; \ + typedef typename Eigen::internal::traits<Derived >::StorageKind StorageKind; \ + typedef typename Eigen::internal::traits<Derived >::Index Index; \ + enum { RowsAtCompileTime = Eigen::internal::traits<Derived >::RowsAtCompileTime, \ + ColsAtCompileTime = Eigen::internal::traits<Derived >::ColsAtCompileTime, \ + Flags = Eigen::internal::traits<Derived>::Flags, \ + SizeAtCompileTime = Base::SizeAtCompileTime, \ + IsVectorAtCompileTime = Base::IsVectorAtCompileTime }; \ + using Base::derived; \ + using Base::const_cast_derived; + +#endif // EIGEN_TEST_EVALUATORS + #define EIGEN_SPARSE_PUBLIC_INTERFACE(Derived) \ _EIGEN_SPARSE_PUBLIC_INTERFACE(Derived, Eigen::SparseMatrixBase<Derived >) @@ -73,7 +94,6 @@ template<typename _Scalar, int _Flags = 0, typename _Index = int> class Dynamic template<typename _Scalar, int _Flags = 0, typename _Index = int> class SparseVector; template<typename _Scalar, int _Flags = 0, typename _Index = int> class MappedSparseMatrix; -template<typename MatrixType, int Mode> class SparseTriangularView; template<typename MatrixType, unsigned int UpLo> class SparseSelfAdjointView; template<typename Lhs, typename Rhs> class SparseDiagonalProduct; template<typename MatrixType> class SparseView; @@ -131,11 +151,23 @@ template<typename T> struct plain_matrix_type<T,Sparse> { typedef typename traits<T>::Scalar _Scalar; typedef typename traits<T>::Index _Index; - enum { _Options = ((traits<T>::Flags&RowMajorBit)==RowMajorBit) ? RowMajor : ColMajor }; + enum { _Options = ((evaluator<T>::Flags&RowMajorBit)==RowMajorBit) ? RowMajor : ColMajor }; public: typedef SparseMatrix<_Scalar, _Options, _Index> type; }; +template<typename Derived> +struct generic_xpr_base<Derived, MatrixXpr, Sparse> +{ + typedef SparseMatrixBase<Derived> type; +}; + +struct SparseTriangularShape { static std::string debugName() { return "SparseTriangularShape"; } }; +struct SparseSelfAdjointShape { static std::string debugName() { return "SparseSelfAdjointShape"; } }; + +template<> struct glue_shapes<SparseShape,SelfAdjointShape> { typedef SparseSelfAdjointShape type; }; +template<> struct glue_shapes<SparseShape,TriangularShape > { typedef SparseTriangularShape type; }; + } // end namespace internal /** \ingroup SparseCore_Module diff --git a/Eigen/src/SparseCore/SparseVector.h b/Eigen/src/SparseCore/SparseVector.h index 0b1b389ce..0f9aa9dd1 100644 --- a/Eigen/src/SparseCore/SparseVector.h +++ b/Eigen/src/SparseCore/SparseVector.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2008-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // // This Source Code Form is subject to the terms of the Mozilla // Public License v. 2.0. If a copy of the MPL was not distributed @@ -422,6 +422,7 @@ class SparseVector<Scalar,_Options,_Index>::ReverseInnerIterator namespace internal { +#ifndef EIGEN_TEST_EVALUATORS template< typename Dest, typename Src> struct sparse_vector_assign_selector<Dest,Src,SVA_Inner> { static void run(Dest& dst, const Src& src) { @@ -443,6 +444,55 @@ struct sparse_vector_assign_selector<Dest,Src,SVA_Outer> { } } }; +#else // EIGEN_TEST_EVALUATORS + +template<typename _Scalar, int _Options, typename _Index> +struct evaluator<SparseVector<_Scalar,_Options,_Index> > + : evaluator_base<SparseVector<_Scalar,_Options,_Index> > +{ + typedef SparseVector<_Scalar,_Options,_Index> SparseVectorType; + typedef typename SparseVectorType::InnerIterator InnerIterator; + typedef typename SparseVectorType::ReverseInnerIterator ReverseInnerIterator; + + enum { + CoeffReadCost = NumTraits<_Scalar>::ReadCost, + Flags = SparseVectorType::Flags + }; + + evaluator(const SparseVectorType &mat) : m_matrix(mat) {} + + operator SparseVectorType&() { return m_matrix.const_cast_derived(); } + operator const SparseVectorType&() const { return m_matrix; } + + const SparseVectorType &m_matrix; +}; + +template< typename Dest, typename Src> +struct sparse_vector_assign_selector<Dest,Src,SVA_Inner> { + static void run(Dest& dst, const Src& src) { + eigen_internal_assert(src.innerSize()==src.size()); + typedef typename internal::evaluator<Src>::type SrcEvaluatorType; + SrcEvaluatorType srcEval(src); + for(typename SrcEvaluatorType::InnerIterator it(srcEval, 0); it; ++it) + dst.insert(it.index()) = it.value(); + } +}; + +template< typename Dest, typename Src> +struct sparse_vector_assign_selector<Dest,Src,SVA_Outer> { + static void run(Dest& dst, const Src& src) { + eigen_internal_assert(src.outerSize()==src.size()); + typedef typename internal::evaluator<Src>::type SrcEvaluatorType; + SrcEvaluatorType srcEval(src); + for(typename Dest::Index i=0; i<src.size(); ++i) + { + typename SrcEvaluatorType::InnerIterator it(srcEval, i); + if(it) + dst.insert(i) = it.value(); + } + } +}; +#endif // EIGEN_TEST_EVALUATORS template< typename Dest, typename Src> struct sparse_vector_assign_selector<Dest,Src,SVA_RuntimeSwitch> { diff --git a/Eigen/src/SparseCore/SparseView.h b/Eigen/src/SparseCore/SparseView.h index fd8450463..c1c50f6e9 100644 --- a/Eigen/src/SparseCore/SparseView.h +++ b/Eigen/src/SparseCore/SparseView.h @@ -1,7 +1,7 @@ // This file is part of Eigen, a lightweight C++ template library // for linear algebra. // -// Copyright (C) 2011 Gael Guennebaud <gael.guennebaud@inria.fr> +// Copyright (C) 2011-2014 Gael Guennebaud <gael.guennebaud@inria.fr> // Copyright (C) 2010 Daniel Lowengrub <lowdanie@gmail.com> // // This Source Code Form is subject to the terms of the Mozilla @@ -34,25 +34,36 @@ class SparseView : public SparseMatrixBase<SparseView<MatrixType> > typedef typename internal::remove_all<MatrixTypeNested>::type _MatrixTypeNested; public: EIGEN_SPARSE_PUBLIC_INTERFACE(SparseView) + typedef typename internal::remove_all<MatrixType>::type NestedExpression; SparseView(const MatrixType& mat, const Scalar& m_reference = Scalar(0), - typename NumTraits<Scalar>::Real m_epsilon = NumTraits<Scalar>::dummy_precision()) : + RealScalar m_epsilon = NumTraits<Scalar>::dummy_precision()) : m_matrix(mat), m_reference(m_reference), m_epsilon(m_epsilon) {} +#ifndef EIGEN_TEST_EVALUATORS class InnerIterator; +#endif // EIGEN_TEST_EVALUATORS inline Index rows() const { return m_matrix.rows(); } inline Index cols() const { return m_matrix.cols(); } inline Index innerSize() const { return m_matrix.innerSize(); } inline Index outerSize() const { return m_matrix.outerSize(); } - + + /** \returns the nested expression */ + const typename internal::remove_all<MatrixTypeNested>::type& + nestedExpression() const { return m_matrix; } + + Scalar reference() const { return m_reference; } + RealScalar epsilon() const { return m_epsilon; } + protected: MatrixTypeNested m_matrix; Scalar m_reference; - typename NumTraits<Scalar>::Real m_epsilon; + RealScalar m_epsilon; }; +#ifndef EIGEN_TEST_EVALUATORS template<typename MatrixType> class SparseView<MatrixType>::InnerIterator : public _MatrixTypeNested::InnerIterator { @@ -80,18 +91,172 @@ protected: private: void incrementToNonZero() { - while((bool(*this)) && internal::isMuchSmallerThan(value(), m_view.m_reference, m_view.m_epsilon)) + while((bool(*this)) && internal::isMuchSmallerThan(value(), m_view.reference(), m_view.epsilon())) { IterBase::operator++(); } } }; +#else // EIGEN_TEST_EVALUATORS + +namespace internal { + +// TODO find a way to unify the two following variants +// This is tricky because implementing an inner iterator on top of an IndexBased evaluator is +// not easy because the evaluators do not expose the sizes of the underlying expression. + +template<typename ArgType> +struct unary_evaluator<SparseView<ArgType>, IteratorBased> + : public evaluator_base<SparseView<ArgType> > +{ + typedef typename evaluator<ArgType>::InnerIterator EvalIterator; + public: + typedef SparseView<ArgType> XprType; + + class InnerIterator : public EvalIterator + { + typedef typename XprType::Scalar Scalar; + public: + + EIGEN_STRONG_INLINE InnerIterator(const unary_evaluator& sve, typename XprType::Index outer) + : EvalIterator(sve.m_argImpl,outer), m_view(sve.m_view) + { + incrementToNonZero(); + } + + EIGEN_STRONG_INLINE InnerIterator& operator++() + { + EvalIterator::operator++(); + incrementToNonZero(); + return *this; + } + + using EvalIterator::value; + + protected: + const XprType &m_view; + + private: + void incrementToNonZero() + { + while((bool(*this)) && internal::isMuchSmallerThan(value(), m_view.reference(), m_view.epsilon())) + { + EvalIterator::operator++(); + } + } + }; + + enum { + CoeffReadCost = evaluator<ArgType>::CoeffReadCost, + Flags = XprType::Flags + }; + + unary_evaluator(const XprType& xpr) : m_argImpl(xpr.nestedExpression()), m_view(xpr) {} + + protected: + typename evaluator<ArgType>::nestedType m_argImpl; + const XprType &m_view; +}; + +template<typename ArgType> +struct unary_evaluator<SparseView<ArgType>, IndexBased> + : public evaluator_base<SparseView<ArgType> > +{ + public: + typedef SparseView<ArgType> XprType; + protected: + enum { IsRowMajor = (XprType::Flags&RowMajorBit)==RowMajorBit }; + typedef typename XprType::Index Index; + typedef typename XprType::Scalar Scalar; + public: + + class InnerIterator + { + public: + + EIGEN_STRONG_INLINE InnerIterator(const unary_evaluator& sve, typename XprType::Index outer) + : m_sve(sve), m_inner(0), m_outer(outer), m_end(sve.m_view.innerSize()) + { + incrementToNonZero(); + } + + EIGEN_STRONG_INLINE InnerIterator& operator++() + { + m_inner++; + incrementToNonZero(); + return *this; + } + + EIGEN_STRONG_INLINE Scalar value() const + { + return (IsRowMajor) ? m_sve.m_argImpl.coeff(m_outer, m_inner) + : m_sve.m_argImpl.coeff(m_inner, m_outer); + } + + EIGEN_STRONG_INLINE Index index() const { return m_inner; } + inline Index row() const { return IsRowMajor ? m_outer : index(); } + inline Index col() const { return IsRowMajor ? index() : m_outer; } + + EIGEN_STRONG_INLINE operator bool() const { return m_inner < m_end && m_inner>=0; } + + protected: + const unary_evaluator &m_sve; + Index m_inner; + const Index m_outer; + const Index m_end; + + private: + void incrementToNonZero() + { + while((bool(*this)) && internal::isMuchSmallerThan(value(), m_sve.m_view.reference(), m_sve.m_view.epsilon())) + { + m_inner++; + } + } + }; + + enum { + CoeffReadCost = evaluator<ArgType>::CoeffReadCost, + Flags = XprType::Flags + }; + + unary_evaluator(const XprType& xpr) : m_argImpl(xpr.nestedExpression()), m_view(xpr) {} + + protected: + typename evaluator<ArgType>::nestedType m_argImpl; + const XprType &m_view; +}; + +} // end namespace internal + +#endif // EIGEN_TEST_EVALUATORS + +template<typename Derived> +const SparseView<Derived> MatrixBase<Derived>::sparseView(const Scalar& reference, + const typename NumTraits<Scalar>::Real& epsilon) const +{ + return SparseView<Derived>(derived(), reference, epsilon); +} + +/** \returns an expression of \c *this with values smaller than + * \a reference * \a epsilon are removed. + * + * This method is typically used in conjunction with the product of two sparse matrices + * to automatically prune the smallest values as follows: + * \code + * C = (A*B).pruned(); // suppress numerical zeros (exact) + * C = (A*B).pruned(ref); + * C = (A*B).pruned(ref,epsilon); + * \endcode + * where \c ref is a meaningful non zero reference value. + * */ template<typename Derived> -const SparseView<Derived> MatrixBase<Derived>::sparseView(const Scalar& m_reference, - const typename NumTraits<Scalar>::Real& m_epsilon) const +const SparseView<Derived> +SparseMatrixBase<Derived>::pruned(const Scalar& reference, + const RealScalar& epsilon) const { - return SparseView<Derived>(derived(), m_reference, m_epsilon); + return SparseView<Derived>(derived(), reference, epsilon); } } // end namespace Eigen diff --git a/Eigen/src/SparseCore/TriangularSolver.h b/Eigen/src/SparseCore/TriangularSolver.h index dd55522a7..012a1bb75 100644 --- a/Eigen/src/SparseCore/TriangularSolver.h +++ b/Eigen/src/SparseCore/TriangularSolver.h @@ -23,6 +23,7 @@ template<typename Lhs, typename Rhs, int Mode, int StorageOrder = int(traits<Lhs>::Flags) & RowMajorBit> struct sparse_solve_triangular_selector; +#ifndef EIGEN_TEST_EVALUATORS // forward substitution, row-major template<typename Lhs, typename Rhs, int Mode> struct sparse_solve_triangular_selector<Lhs,Rhs,Mode,Lower,RowMajor> @@ -163,13 +164,166 @@ struct sparse_solve_triangular_selector<Lhs,Rhs,Mode,Upper,ColMajor> } }; +#else // EIGEN_TEST_EVALUATORS + +// forward substitution, row-major +template<typename Lhs, typename Rhs, int Mode> +struct sparse_solve_triangular_selector<Lhs,Rhs,Mode,Lower,RowMajor> +{ + typedef typename Rhs::Scalar Scalar; + typedef typename Lhs::Index Index; + typedef typename evaluator<Lhs>::type LhsEval; + typedef typename evaluator<Lhs>::InnerIterator LhsIterator; + static void run(const Lhs& lhs, Rhs& other) + { + LhsEval lhsEval(lhs); + for(Index col=0 ; col<other.cols() ; ++col) + { + for(Index i=0; i<lhs.rows(); ++i) + { + Scalar tmp = other.coeff(i,col); + Scalar lastVal(0); + Index lastIndex = 0; + for(LhsIterator it(lhsEval, i); it; ++it) + { + lastVal = it.value(); + lastIndex = it.index(); + if(lastIndex==i) + break; + tmp -= lastVal * other.coeff(lastIndex,col); + } + if (Mode & UnitDiag) + other.coeffRef(i,col) = tmp; + else + { + eigen_assert(lastIndex==i); + other.coeffRef(i,col) = tmp/lastVal; + } + } + } + } +}; + +// backward substitution, row-major +template<typename Lhs, typename Rhs, int Mode> +struct sparse_solve_triangular_selector<Lhs,Rhs,Mode,Upper,RowMajor> +{ + typedef typename Rhs::Scalar Scalar; + typedef typename Lhs::Index Index; + typedef typename evaluator<Lhs>::type LhsEval; + typedef typename evaluator<Lhs>::InnerIterator LhsIterator; + static void run(const Lhs& lhs, Rhs& other) + { + LhsEval lhsEval(lhs); + for(Index col=0 ; col<other.cols() ; ++col) + { + for(Index i=lhs.rows()-1 ; i>=0 ; --i) + { + Scalar tmp = other.coeff(i,col); + Scalar l_ii = 0; + LhsIterator it(lhsEval, i); + while(it && it.index()<i) + ++it; + if(!(Mode & UnitDiag)) + { + eigen_assert(it && it.index()==i); + l_ii = it.value(); + ++it; + } + else if (it && it.index() == i) + ++it; + for(; it; ++it) + { + tmp -= it.value() * other.coeff(it.index(),col); + } + + if (Mode & UnitDiag) other.coeffRef(i,col) = tmp; + else other.coeffRef(i,col) = tmp/l_ii; + } + } + } +}; + +// forward substitution, col-major +template<typename Lhs, typename Rhs, int Mode> +struct sparse_solve_triangular_selector<Lhs,Rhs,Mode,Lower,ColMajor> +{ + typedef typename Rhs::Scalar Scalar; + typedef typename Lhs::Index Index; + typedef typename evaluator<Lhs>::type LhsEval; + typedef typename evaluator<Lhs>::InnerIterator LhsIterator; + static void run(const Lhs& lhs, Rhs& other) + { + LhsEval lhsEval(lhs); + for(Index col=0 ; col<other.cols() ; ++col) + { + for(Index i=0; i<lhs.cols(); ++i) + { + Scalar& tmp = other.coeffRef(i,col); + if (tmp!=Scalar(0)) // optimization when other is actually sparse + { + LhsIterator it(lhsEval, i); + while(it && it.index()<i) + ++it; + if(!(Mode & UnitDiag)) + { + eigen_assert(it && it.index()==i); + tmp /= it.value(); + } + if (it && it.index()==i) + ++it; + for(; it; ++it) + other.coeffRef(it.index(), col) -= tmp * it.value(); + } + } + } + } +}; + +// backward substitution, col-major +template<typename Lhs, typename Rhs, int Mode> +struct sparse_solve_triangular_selector<Lhs,Rhs,Mode,Upper,ColMajor> +{ + typedef typename Rhs::Scalar Scalar; + typedef typename Lhs::Index Index; + typedef typename evaluator<Lhs>::type LhsEval; + typedef typename evaluator<Lhs>::InnerIterator LhsIterator; + static void run(const Lhs& lhs, Rhs& other) + { + LhsEval lhsEval(lhs); + for(Index col=0 ; col<other.cols() ; ++col) + { + for(Index i=lhs.cols()-1; i>=0; --i) + { + Scalar& tmp = other.coeffRef(i,col); + if (tmp!=Scalar(0)) // optimization when other is actually sparse + { + if(!(Mode & UnitDiag)) + { + // TODO replace this by a binary search. make sure the binary search is safe for partially sorted elements + LhsIterator it(lhsEval, i); + while(it && it.index()!=i) + ++it; + eigen_assert(it && it.index()==i); + other.coeffRef(i,col) /= it.value(); + } + LhsIterator it(lhsEval, i); + for(; it && it.index()<i; ++it) + other.coeffRef(it.index(), col) -= tmp * it.value(); + } + } + } + } +}; + +#endif // EIGEN_TEST_EVALUATORS } // end namespace internal -template<typename ExpressionType,int Mode> +template<typename ExpressionType,unsigned int Mode> template<typename OtherDerived> -void SparseTriangularView<ExpressionType,Mode>::solveInPlace(MatrixBase<OtherDerived>& other) const +void TriangularViewImpl<ExpressionType,Mode,Sparse>::solveInPlace(MatrixBase<OtherDerived>& other) const { - eigen_assert(m_matrix.cols() == m_matrix.rows() && m_matrix.cols() == other.rows()); + eigen_assert(derived().cols() == derived().rows() && derived().cols() == other.rows()); eigen_assert((!(Mode & ZeroDiag)) && bool(Mode & (Upper|Lower))); enum { copy = internal::traits<OtherDerived>::Flags & RowMajorBit }; @@ -178,21 +332,23 @@ void SparseTriangularView<ExpressionType,Mode>::solveInPlace(MatrixBase<OtherDer typename internal::plain_matrix_type_column_major<OtherDerived>::type, OtherDerived&>::type OtherCopy; OtherCopy otherCopy(other.derived()); - internal::sparse_solve_triangular_selector<ExpressionType, typename internal::remove_reference<OtherCopy>::type, Mode>::run(m_matrix, otherCopy); + internal::sparse_solve_triangular_selector<ExpressionType, typename internal::remove_reference<OtherCopy>::type, Mode>::run(derived().nestedExpression(), otherCopy); if (copy) other = otherCopy; } -template<typename ExpressionType,int Mode> +#ifndef EIGEN_TEST_EVALUATORS +template<typename ExpressionType,unsigned int Mode> template<typename OtherDerived> typename internal::plain_matrix_type_column_major<OtherDerived>::type -SparseTriangularView<ExpressionType,Mode>::solve(const MatrixBase<OtherDerived>& other) const +TriangularViewImpl<ExpressionType,Mode,Sparse>::solve(const MatrixBase<OtherDerived>& other) const { typename internal::plain_matrix_type_column_major<OtherDerived>::type res(other); solveInPlace(res); return res; } +#endif // pure sparse path @@ -290,11 +446,11 @@ struct sparse_solve_triangular_sparse_selector<Lhs,Rhs,Mode,UpLo,ColMajor> } // end namespace internal -template<typename ExpressionType,int Mode> +template<typename ExpressionType,unsigned int Mode> template<typename OtherDerived> -void SparseTriangularView<ExpressionType,Mode>::solveInPlace(SparseMatrixBase<OtherDerived>& other) const +void TriangularViewImpl<ExpressionType,Mode,Sparse>::solveInPlace(SparseMatrixBase<OtherDerived>& other) const { - eigen_assert(m_matrix.cols() == m_matrix.rows() && m_matrix.cols() == other.rows()); + eigen_assert(derived().cols() == derived().rows() && derived().cols() == other.rows()); eigen_assert( (!(Mode & ZeroDiag)) && bool(Mode & (Upper|Lower))); // enum { copy = internal::traits<OtherDerived>::Flags & RowMajorBit }; @@ -303,7 +459,7 @@ void SparseTriangularView<ExpressionType,Mode>::solveInPlace(SparseMatrixBase<Ot // typename internal::plain_matrix_type_column_major<OtherDerived>::type, OtherDerived&>::type OtherCopy; // OtherCopy otherCopy(other.derived()); - internal::sparse_solve_triangular_sparse_selector<ExpressionType, OtherDerived, Mode>::run(m_matrix, other.derived()); + internal::sparse_solve_triangular_sparse_selector<ExpressionType, OtherDerived, Mode>::run(derived().nestedExpression(), other.derived()); // if (copy) // other = otherCopy; diff --git a/Eigen/src/SparseQR/SparseQR.h b/Eigen/src/SparseQR/SparseQR.h index 002b4824b..6be569533 100644 --- a/Eigen/src/SparseQR/SparseQR.h +++ b/Eigen/src/SparseQR/SparseQR.h @@ -178,7 +178,7 @@ class SparseQR y.resize((std::max)(cols(),Index(y.rows())),y.cols()); y.topRows(rank) = this->matrixR().topLeftCorner(rank, rank).template triangularView<Upper>().solve(b.topRows(rank)); y.bottomRows(y.rows()-rank).setZero(); - + // Apply the column permutation if (m_perm_c.size()) dest = colsPermutation() * y.topRows(cols()); else dest = y.topRows(cols()); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 47aefddb8..6446b8bb0 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -142,7 +142,6 @@ add_custom_target(BuildOfficial) option(EIGEN_TEST_EVALUATORS "Enable work in progress evaluators" OFF) if(EIGEN_TEST_EVALUATORS) add_definitions("-DEIGEN_TEST_EVALUATORS=1") - add_definitions("-DEIGEN_ENABLE_EVALUATORS=1") endif(EIGEN_TEST_EVALUATORS) ei_add_test(meta) @@ -150,6 +149,7 @@ ei_add_test(sizeof) ei_add_test(dynalloc) ei_add_test(nomalloc) ei_add_test(first_aligned) +ei_add_test(nullary) ei_add_test(mixingtypes) ei_add_test(packetmath) ei_add_test(unalignedassert) @@ -165,6 +165,9 @@ ei_add_test(redux) ei_add_test(visitor) ei_add_test(block) ei_add_test(corners) +ei_add_test(swap) +ei_add_test(resize) +ei_add_test(conservative_resize) ei_add_test(product_small) ei_add_test(product_large) ei_add_test(product_extra) @@ -193,6 +196,7 @@ ei_add_test(product_trsolve) ei_add_test(product_mmtr) ei_add_test(product_notemporary) ei_add_test(stable_norm) +ei_add_test(permutationmatrices) ei_add_test(bandmatrix) ei_add_test(cholesky) ei_add_test(lu) @@ -212,30 +216,30 @@ ei_add_test(real_qz) ei_add_test(eigensolver_generalized_real) ei_add_test(jacobi) ei_add_test(jacobisvd) +ei_add_test(householder) ei_add_test(geo_orthomethods) -ei_add_test(geo_homogeneous) ei_add_test(geo_quaternion) -ei_add_test(geo_transformations) ei_add_test(geo_eulerangles) -ei_add_test(geo_hyperplane) ei_add_test(geo_parametrizedline) ei_add_test(geo_alignedbox) +ei_add_test(geo_hyperplane) +ei_add_test(geo_transformations) +ei_add_test(geo_homogeneous) ei_add_test(stdvector) ei_add_test(stdvector_overload) ei_add_test(stdlist) ei_add_test(stddeque) -ei_add_test(resize) -ei_add_test(sparse_vector) ei_add_test(sparse_basic) +ei_add_test(sparse_vector) ei_add_test(sparse_product) ei_add_test(sparse_solvers) -ei_add_test(umeyama) -ei_add_test(householder) -ei_add_test(swap) -ei_add_test(conservative_resize) -ei_add_test(permutationmatrices) ei_add_test(sparse_permutations) -ei_add_test(nullary) +ei_add_test(simplicial_cholesky) +ei_add_test(conjugate_gradient) +ei_add_test(bicgstab) +ei_add_test(sparselu) +ei_add_test(sparseqr) +ei_add_test(umeyama) ei_add_test(nesting_ops "${CMAKE_CXX_FLAGS_DEBUG}") ei_add_test(zerosized) ei_add_test(dontalign) @@ -249,13 +253,7 @@ ei_add_test(special_numbers) ei_add_test(rvalue_types) ei_add_test(dense_storage) -ei_add_test(simplicial_cholesky) -ei_add_test(conjugate_gradient) -ei_add_test(bicgstab) -ei_add_test(sparselu) -ei_add_test(sparseqr) - -# ei_add_test(denseLM) +# # ei_add_test(denseLM) if(QT4_FOUND) ei_add_test(qtvector "" "${QT_QTCORE_LIBRARY}") diff --git a/test/evaluators.cpp b/test/evaluators.cpp index e3922c1be..305047a6a 100644 --- a/test/evaluators.cpp +++ b/test/evaluators.cpp @@ -1,7 +1,86 @@ + +#ifndef EIGEN_ENABLE_EVALUATORS #define EIGEN_ENABLE_EVALUATORS +#endif + +#ifdef EIGEN_TEST_EVALUATORS +#undef EIGEN_TEST_EVALUATORS +#endif + #include "main.h" -using internal::copy_using_evaluator; +namespace Eigen { + + template<typename DstXprType, typename SrcXprType> + EIGEN_STRONG_INLINE + DstXprType& copy_using_evaluator(const EigenBase<DstXprType> &dst, const SrcXprType &src) + { + call_assignment(dst.const_cast_derived(), src.derived(), internal::assign_op<typename DstXprType::Scalar>()); + return dst.const_cast_derived(); + } + + template<typename DstXprType, template <typename> class StorageBase, typename SrcXprType> + EIGEN_STRONG_INLINE + const DstXprType& copy_using_evaluator(const NoAlias<DstXprType, StorageBase>& dst, const SrcXprType &src) + { + call_assignment(dst, src.derived(), internal::assign_op<typename DstXprType::Scalar>()); + return dst.expression(); + } + + template<typename DstXprType, typename SrcXprType> + EIGEN_STRONG_INLINE + DstXprType& copy_using_evaluator(const PlainObjectBase<DstXprType> &dst, const SrcXprType &src) + { + #ifdef EIGEN_NO_AUTOMATIC_RESIZING + eigen_assert((dst.size()==0 || (IsVectorAtCompileTime ? (dst.size() == src.size()) + : (dst.rows() == src.rows() && dst.cols() == src.cols()))) + && "Size mismatch. Automatic resizing is disabled because EIGEN_NO_AUTOMATIC_RESIZING is defined"); + #else + dst.const_cast_derived().resizeLike(src.derived()); + #endif + + call_assignment(dst.const_cast_derived(), src.derived(), internal::assign_op<typename DstXprType::Scalar>()); + return dst.const_cast_derived(); + } + + template<typename DstXprType, typename SrcXprType> + void add_assign_using_evaluator(const DstXprType& dst, const SrcXprType& src) + { + typedef typename DstXprType::Scalar Scalar; + call_assignment(const_cast<DstXprType&>(dst), src.derived(), internal::add_assign_op<Scalar>()); + } + + template<typename DstXprType, typename SrcXprType> + void subtract_assign_using_evaluator(const DstXprType& dst, const SrcXprType& src) + { + typedef typename DstXprType::Scalar Scalar; + call_assignment(const_cast<DstXprType&>(dst), src.derived(), internal::sub_assign_op<Scalar>()); + } + + template<typename DstXprType, typename SrcXprType> + void multiply_assign_using_evaluator(const DstXprType& dst, const SrcXprType& src) + { + typedef typename DstXprType::Scalar Scalar; + call_assignment(dst.const_cast_derived(), src.derived(), internal::mul_assign_op<Scalar>()); + } + + template<typename DstXprType, typename SrcXprType> + void divide_assign_using_evaluator(const DstXprType& dst, const SrcXprType& src) + { + typedef typename DstXprType::Scalar Scalar; + call_assignment(dst.const_cast_derived(), src.derived(), internal::div_assign_op<Scalar>()); + } + + template<typename DstXprType, typename SrcXprType> + void swap_using_evaluator(const DstXprType& dst, const SrcXprType& src) + { + typedef typename DstXprType::Scalar Scalar; + call_assignment(dst.const_cast_derived(), src.const_cast_derived(), internal::swap_assign_op<Scalar>()); + } + +} + + using namespace std; #define VERIFY_IS_APPROX_EVALUATOR(DEST,EXPR) VERIFY_IS_APPROX(copy_using_evaluator(DEST,(EXPR)), (EXPR).eval()); @@ -72,8 +151,19 @@ void test_evaluators() c = a*a; copy_using_evaluator(a, prod(a,a)); VERIFY_IS_APPROX(a,c); + + // check compound assignment of products + d = c; + add_assign_using_evaluator(c.noalias(), prod(a,b)); + d.noalias() += a*b; + VERIFY_IS_APPROX(c, d); + + d = c; + subtract_assign_using_evaluator(c.noalias(), prod(a,b)); + d.noalias() -= a*b; + VERIFY_IS_APPROX(c, d); } - + { // test product with all possible sizes int s = internal::random<int>(1,100); @@ -124,7 +214,7 @@ void test_evaluators() // this does not work because Random is eval-before-nested: // copy_using_evaluator(w, Vector2d::Random().transpose()); - + // test CwiseUnaryOp VERIFY_IS_APPROX_EVALUATOR(v2, 3 * v); VERIFY_IS_APPROX_EVALUATOR(w, (3 * v).transpose()); @@ -327,4 +417,56 @@ void test_evaluators() arr_ref.row(1) /= (arr_ref.row(2) + 1); VERIFY_IS_APPROX(arr, arr_ref); } + + { + // test triangular shapes + MatrixXd A = MatrixXd::Random(6,6), B(6,6), C(6,6), D(6,6); + A.setRandom();B.setRandom(); + VERIFY_IS_APPROX_EVALUATOR2(B, A.triangularView<Upper>(), MatrixXd(A.triangularView<Upper>())); + + A.setRandom();B.setRandom(); + VERIFY_IS_APPROX_EVALUATOR2(B, A.triangularView<UnitLower>(), MatrixXd(A.triangularView<UnitLower>())); + + A.setRandom();B.setRandom(); + VERIFY_IS_APPROX_EVALUATOR2(B, A.triangularView<UnitUpper>(), MatrixXd(A.triangularView<UnitUpper>())); + + A.setRandom();B.setRandom(); + C = B; C.triangularView<Upper>() = A; + copy_using_evaluator(B.triangularView<Upper>(), A); + VERIFY(B.isApprox(C) && "copy_using_evaluator(B.triangularView<Upper>(), A)"); + + A.setRandom();B.setRandom(); + C = B; C.triangularView<Lower>() = A.triangularView<Lower>(); + copy_using_evaluator(B.triangularView<Lower>(), A.triangularView<Lower>()); + VERIFY(B.isApprox(C) && "copy_using_evaluator(B.triangularView<Lower>(), A.triangularView<Lower>())"); + + + A.setRandom();B.setRandom(); + C = B; C.triangularView<Lower>() = A.triangularView<Upper>().transpose(); + copy_using_evaluator(B.triangularView<Lower>(), A.triangularView<Upper>().transpose()); + VERIFY(B.isApprox(C) && "copy_using_evaluator(B.triangularView<Lower>(), A.triangularView<Lower>().transpose())"); + + + A.setRandom();B.setRandom(); C = B; D = A; + C.triangularView<Upper>().swap(D.triangularView<Upper>()); + swap_using_evaluator(B.triangularView<Upper>(), A.triangularView<Upper>()); + VERIFY(B.isApprox(C) && "swap_using_evaluator(B.triangularView<Upper>(), A.triangularView<Upper>())"); + + + VERIFY_IS_APPROX_EVALUATOR2(B, prod(A.triangularView<Upper>(),A), MatrixXd(A.triangularView<Upper>()*A)); + + VERIFY_IS_APPROX_EVALUATOR2(B, prod(A.selfadjointView<Upper>(),A), MatrixXd(A.selfadjointView<Upper>()*A)); + + } + + { + // test diagonal shapes + VectorXd d = VectorXd::Random(6); + MatrixXd A = MatrixXd::Random(6,6), B(6,6); + A.setRandom();B.setRandom(); + + VERIFY_IS_APPROX_EVALUATOR2(B, lazyprod(d.asDiagonal(),A), MatrixXd(d.asDiagonal()*A)); + VERIFY_IS_APPROX_EVALUATOR2(B, lazyprod(A,d.asDiagonal()), MatrixXd(A*d.asDiagonal())); + + } } diff --git a/test/geo_homogeneous.cpp b/test/geo_homogeneous.cpp index c91bde819..078894035 100644 --- a/test/geo_homogeneous.cpp +++ b/test/geo_homogeneous.cpp @@ -57,7 +57,6 @@ template<typename Scalar,int Size> void homogeneous(void) VERIFY_IS_APPROX((v0.transpose().rowwise().homogeneous().eval()) * t2, v0.transpose().rowwise().homogeneous() * t2); - m0.transpose().rowwise().homogeneous().eval(); VERIFY_IS_APPROX((m0.transpose().rowwise().homogeneous().eval()) * t2, m0.transpose().rowwise().homogeneous() * t2); @@ -82,7 +81,7 @@ template<typename Scalar,int Size> void homogeneous(void) VERIFY_IS_APPROX(aff * pts.colwise().homogeneous(), (aff * pts1).colwise().hnormalized()); VERIFY_IS_APPROX(caff * pts.colwise().homogeneous(), (caff * pts1).colwise().hnormalized()); VERIFY_IS_APPROX(proj * pts.colwise().homogeneous(), (proj * pts1)); - + VERIFY_IS_APPROX((aff * pts1).colwise().hnormalized(), aff * pts); VERIFY_IS_APPROX((caff * pts1).colwise().hnormalized(), caff * pts); diff --git a/test/geo_orthomethods.cpp b/test/geo_orthomethods.cpp index c836dae40..7f8beb205 100644 --- a/test/geo_orthomethods.cpp +++ b/test/geo_orthomethods.cpp @@ -33,6 +33,7 @@ template<typename Scalar> void orthomethods_3() VERIFY_IS_MUCH_SMALLER_THAN(v1.dot(v1.cross(v2)), Scalar(1)); VERIFY_IS_MUCH_SMALLER_THAN(v1.cross(v2).dot(v2), Scalar(1)); VERIFY_IS_MUCH_SMALLER_THAN(v2.dot(v1.cross(v2)), Scalar(1)); + VERIFY_IS_MUCH_SMALLER_THAN(v1.cross(Vector3::Random()).dot(v1), Scalar(1)); Matrix3 mat3; mat3 << v0.normalized(), (v0.cross(v1)).normalized(), @@ -47,6 +48,13 @@ template<typename Scalar> void orthomethods_3() int i = internal::random<int>(0,2); mcross = mat3.colwise().cross(vec3); VERIFY_IS_APPROX(mcross.col(i), mat3.col(i).cross(vec3)); + + VERIFY_IS_MUCH_SMALLER_THAN((mat3.adjoint() * mat3.colwise().cross(vec3)).diagonal().cwiseAbs().sum(), Scalar(1)); + VERIFY_IS_MUCH_SMALLER_THAN((mat3.adjoint() * mat3.colwise().cross(Vector3::Random())).diagonal().cwiseAbs().sum(), Scalar(1)); + + VERIFY_IS_MUCH_SMALLER_THAN((vec3.adjoint() * mat3.colwise().cross(vec3)).cwiseAbs().sum(), Scalar(1)); + VERIFY_IS_MUCH_SMALLER_THAN((vec3.adjoint() * Matrix3::Random().colwise().cross(vec3)).cwiseAbs().sum(), Scalar(1)); + mcross = mat3.rowwise().cross(vec3); VERIFY_IS_APPROX(mcross.row(i), mat3.row(i).cross(vec3)); @@ -57,6 +65,7 @@ template<typename Scalar> void orthomethods_3() v40.w() = v41.w() = v42.w() = 0; v42.template head<3>() = v40.template head<3>().cross(v41.template head<3>()); VERIFY_IS_APPROX(v40.cross3(v41), v42); + VERIFY_IS_MUCH_SMALLER_THAN(v40.cross3(Vector4::Random()).dot(v40), Scalar(1)); // check mixed product typedef Matrix<RealScalar, 3, 1> RealVector3; diff --git a/test/inverse.cpp b/test/inverse.cpp index 8187b088d..1195bcc76 100644 --- a/test/inverse.cpp +++ b/test/inverse.cpp @@ -68,6 +68,14 @@ template<typename MatrixType> void inverse(const MatrixType& m) VERIFY_IS_MUCH_SMALLER_THAN(abs(det-m3.determinant()), RealScalar(1)); m3.computeInverseWithCheck(m4, invertible); VERIFY( rows==1 ? invertible : !invertible ); + + // check with submatrices + { + Matrix<Scalar, MatrixType::RowsAtCompileTime+1, MatrixType::RowsAtCompileTime+1, MatrixType::Options> m3; + m3.setRandom(); + m2 = m3.template topLeftCorner<MatrixType::RowsAtCompileTime,MatrixType::ColsAtCompileTime>().inverse(); + VERIFY_IS_APPROX( (m3.template topLeftCorner<MatrixType::RowsAtCompileTime,MatrixType::ColsAtCompileTime>()), m2.inverse() ); + } #endif // check in-place inversion diff --git a/test/main.h b/test/main.h index 376232cf2..b8854a1c3 100644 --- a/test/main.h +++ b/test/main.h @@ -71,6 +71,9 @@ namespace Eigen static bool g_has_set_repeat, g_has_set_seed; } +#define TRACK std::cerr << __FILE__ << " " << __LINE__ << std::endl +// #define TRACK while() + #define EI_PP_MAKE_STRING2(S) #S #define EI_PP_MAKE_STRING(S) EI_PP_MAKE_STRING2(S) @@ -288,13 +291,7 @@ inline bool test_isApproxOrLessThan(const long double& a, const long double& b) template<typename Type1, typename Type2> inline bool test_isApprox(const Type1& a, const Type2& b) { -#ifdef EIGEN_TEST_EVALUATORS - typename internal::eval<Type1>::type a_eval(a); - typename internal::eval<Type2>::type b_eval(b); - return a_eval.isApprox(b_eval, test_precision<typename Type1::Scalar>()); -#else return a.isApprox(b, test_precision<typename Type1::Scalar>()); -#endif } // The idea behind this function is to compare the two scalars a and b where diff --git a/test/mixingtypes.cpp b/test/mixingtypes.cpp index 1e0e2d4c1..976e21e37 100644 --- a/test/mixingtypes.cpp +++ b/test/mixingtypes.cpp @@ -53,10 +53,13 @@ template<int SizeAtCompileType> void mixingtypes(int size = SizeAtCompileType) mf+mf; VERIFY_RAISES_ASSERT(mf+md); VERIFY_RAISES_ASSERT(mf+mcf); +#ifndef EIGEN_TEST_EVALUATORS + // they do not even compile when using evaluators VERIFY_RAISES_ASSERT(vf=vd); VERIFY_RAISES_ASSERT(vf+=vd); VERIFY_RAISES_ASSERT(mcd=md); - +#endif + // check scalar products VERIFY_IS_APPROX(vcf * sf , vcf * complex<float>(sf)); VERIFY_IS_APPROX(sd * vcd, complex<double>(sd) * vcd); diff --git a/test/nesting_ops.cpp b/test/nesting_ops.cpp index 1e8523283..114dd5e41 100644 --- a/test/nesting_ops.cpp +++ b/test/nesting_ops.cpp @@ -11,7 +11,12 @@ template <typename MatrixType> void run_nesting_ops(const MatrixType& _m) { +#ifndef EIGEN_TEST_EVALUATORS + // TODO, with evaluator, the following is not correct anymore: typename MatrixType::Nested m(_m); +#else + typename internal::nested_eval<MatrixType,2>::type m(_m); +#endif // Make really sure that we are in debug mode! VERIFY_RAISES_ASSERT(eigen_assert(false)); diff --git a/test/product_notemporary.cpp b/test/product_notemporary.cpp index 3a9df618b..805cc8939 100644 --- a/test/product_notemporary.cpp +++ b/test/product_notemporary.cpp @@ -113,8 +113,7 @@ template<typename MatrixType> void product_notemporary(const MatrixType& m) VERIFY_EVALUATION_COUNT( Scalar tmp = 0; tmp += Scalar(RealScalar(1)) / (m3.transpose() * m3).diagonal().array().abs().sum(), 0 ); // Zero temporaries for ... CoeffBasedProductMode - // - does not work with GCC because of the <..>, we'ld need variadic macros ... - //VERIFY_EVALUATION_COUNT( m3.col(0).head<5>() * m3.col(0).transpose() + m3.col(0).head<5>() * m3.col(0).transpose(), 0 ); + VERIFY_EVALUATION_COUNT( m3.col(0).template head<5>() * m3.col(0).transpose() + m3.col(0).template head<5>() * m3.col(0).transpose(), 0 ); // Check matrix * vectors VERIFY_EVALUATION_COUNT( cvres.noalias() = m1 * cv1, 0 ); diff --git a/test/qr_fullpivoting.cpp b/test/qr_fullpivoting.cpp index 511f2473f..601773404 100644 --- a/test/qr_fullpivoting.cpp +++ b/test/qr_fullpivoting.cpp @@ -40,7 +40,11 @@ template<typename MatrixType> void qr() MatrixType c = qr.matrixQ() * r * qr.colsPermutation().inverse(); VERIFY_IS_APPROX(m1, c); - + + // stress the ReturnByValue mechanism + MatrixType tmp; + VERIFY_IS_APPROX(tmp.noalias() = qr.matrixQ() * r, (qr.matrixQ() * r).eval()); + MatrixType m2 = MatrixType::Random(cols,cols2); MatrixType m3 = m1*m2; m2 = MatrixType::Random(cols,cols2); diff --git a/test/sparse_basic.cpp b/test/sparse_basic.cpp index 4c9b9111e..c86534bad 100644 --- a/test/sparse_basic.cpp +++ b/test/sparse_basic.cpp @@ -201,9 +201,9 @@ template<typename SparseMatrixType> void sparse_basic(const SparseMatrixType& re VERIFY(m3.innerVector(j0).nonZeros() == m3.transpose().innerVector(j0).nonZeros()); - //m2.innerVector(j0) = 2*m2.innerVector(j1); - //refMat2.col(j0) = 2*refMat2.col(j1); - //VERIFY_IS_APPROX(m2, refMat2); +// m2.innerVector(j0) = 2*m2.innerVector(j1); +// refMat2.col(j0) = 2*refMat2.col(j1); +// VERIFY_IS_APPROX(m2, refMat2); } // test innerVectors() @@ -239,7 +239,7 @@ template<typename SparseMatrixType> void sparse_basic(const SparseMatrixType& re VERIFY_IS_APPROX(m2, refMat2); } - + // test basic computations { DenseMatrix refM1 = DenseMatrix::Zero(rows, rows); @@ -255,6 +255,7 @@ template<typename SparseMatrixType> void sparse_basic(const SparseMatrixType& re initSparse<Scalar>(density, refM3, m3); initSparse<Scalar>(density, refM4, m4); + VERIFY_IS_APPROX(m1*s1, refM1*s1); VERIFY_IS_APPROX(m1+m2, refM1+refM2); VERIFY_IS_APPROX(m1+m2+m3, refM1+refM2+refM3); VERIFY_IS_APPROX(m3.cwiseProduct(m1+m2), refM3.cwiseProduct(refM1+refM2)); diff --git a/test/sparse_product.cpp b/test/sparse_product.cpp index 0f52164c8..fa9be5440 100644 --- a/test/sparse_product.cpp +++ b/test/sparse_product.cpp @@ -19,7 +19,7 @@ template<typename SparseMatrixType> void sparse_product() typedef typename SparseMatrixType::Scalar Scalar; enum { Flags = SparseMatrixType::Flags }; - double density = (std::max)(8./(rows*cols), 0.1); + double density = (std::max)(8./(rows*cols), 0.2); typedef Matrix<Scalar,Dynamic,Dynamic> DenseMatrix; typedef Matrix<Scalar,Dynamic,1> DenseVector; typedef Matrix<Scalar,1,Dynamic> RowDenseVector; @@ -77,17 +77,27 @@ template<typename SparseMatrixType> void sparse_product() m4 = m2; refMat4 = refMat2; VERIFY_IS_APPROX(m4=m4*m3, refMat4=refMat4*refMat3); - // sparse * dense + // sparse * dense matrix VERIFY_IS_APPROX(dm4=m2*refMat3, refMat4=refMat2*refMat3); VERIFY_IS_APPROX(dm4=m2*refMat3t.transpose(), refMat4=refMat2*refMat3t.transpose()); VERIFY_IS_APPROX(dm4=m2t.transpose()*refMat3, refMat4=refMat2t.transpose()*refMat3); VERIFY_IS_APPROX(dm4=m2t.transpose()*refMat3t.transpose(), refMat4=refMat2t.transpose()*refMat3t.transpose()); + VERIFY_IS_APPROX(dm4=m2*refMat3, refMat4=refMat2*refMat3); + VERIFY_IS_APPROX(dm4=dm4+m2*refMat3, refMat4=refMat4+refMat2*refMat3); VERIFY_IS_APPROX(dm4=m2*(refMat3+refMat3), refMat4=refMat2*(refMat3+refMat3)); VERIFY_IS_APPROX(dm4=m2t.transpose()*(refMat3+refMat5)*0.5, refMat4=refMat2t.transpose()*(refMat3+refMat5)*0.5); + + // sparse * dense vector + VERIFY_IS_APPROX(dm4.col(0)=m2*refMat3.col(0), refMat4.col(0)=refMat2*refMat3.col(0)); + VERIFY_IS_APPROX(dm4.col(0)=m2*refMat3t.transpose().col(0), refMat4.col(0)=refMat2*refMat3t.transpose().col(0)); + VERIFY_IS_APPROX(dm4.col(0)=m2t.transpose()*refMat3.col(0), refMat4.col(0)=refMat2t.transpose()*refMat3.col(0)); + VERIFY_IS_APPROX(dm4.col(0)=m2t.transpose()*refMat3t.transpose().col(0), refMat4.col(0)=refMat2t.transpose()*refMat3t.transpose().col(0)); // dense * sparse VERIFY_IS_APPROX(dm4=refMat2*m3, refMat4=refMat2*refMat3); + VERIFY_IS_APPROX(dm4=dm4+refMat2*m3, refMat4=refMat4+refMat2*refMat3); + VERIFY_IS_APPROX(dm4+=refMat2*m3, refMat4+=refMat2*refMat3); VERIFY_IS_APPROX(dm4=refMat2*m3t.transpose(), refMat4=refMat2*refMat3t.transpose()); VERIFY_IS_APPROX(dm4=refMat2t.transpose()*m3, refMat4=refMat2t.transpose()*refMat3); VERIFY_IS_APPROX(dm4=refMat2t.transpose()*m3t.transpose(), refMat4=refMat2t.transpose()*refMat3t.transpose()); @@ -99,7 +109,7 @@ template<typename SparseMatrixType> void sparse_product() Index c1 = internal::random<Index>(0,cols-1); Index r1 = internal::random<Index>(0,depth-1); DenseMatrix dm5 = DenseMatrix::Random(depth, cols); - + VERIFY_IS_APPROX( m4=m2.col(c)*dm5.col(c1).transpose(), refMat4=refMat2.col(c)*dm5.col(c1).transpose()); VERIFY_IS_EQUAL(m4.nonZeros(), (refMat4.array()!=0).count()); VERIFY_IS_APPROX( m4=m2.middleCols(c,1)*dm5.col(c1).transpose(), refMat4=refMat2.col(c)*dm5.col(c1).transpose()); @@ -143,11 +153,11 @@ template<typename SparseMatrixType> void sparse_product() RowSpVector rv0(depth), rv1; RowDenseVector drv0(depth), drv1(rv1); initSparse(2*density,drv0, rv0); - - VERIFY_IS_APPROX(cv1=rv0*m3, dcv1=drv0*refMat3); + + VERIFY_IS_APPROX(cv1=m3*cv0, dcv1=refMat3*dcv0); VERIFY_IS_APPROX(rv1=rv0*m3, drv1=drv0*refMat3); - VERIFY_IS_APPROX(cv1=m3*cv0, dcv1=refMat3*dcv0); VERIFY_IS_APPROX(cv1=m3t.adjoint()*cv0, dcv1=refMat3t.adjoint()*dcv0); + VERIFY_IS_APPROX(cv1=rv0*m3, dcv1=drv0*refMat3); VERIFY_IS_APPROX(rv1=m3*cv0, drv1=refMat3*dcv0); } diff --git a/test/vectorization_logic.cpp b/test/vectorization_logic.cpp index b069f0771..42015e21b 100644 --- a/test/vectorization_logic.cpp +++ b/test/vectorization_logic.cpp @@ -27,19 +27,43 @@ std::string demangle_unrolling(int t) if(t==CompleteUnrolling) return "CompleteUnrolling"; return "?"; } +std::string demangle_flags(int f) +{ + std::string res; + if(f&RowMajorBit) res += " | RowMajor"; + if(f&PacketAccessBit) res += " | Packet"; + if(f&LinearAccessBit) res += " | Linear"; + if(f&LvalueBit) res += " | Lvalue"; + if(f&DirectAccessBit) res += " | Direct"; + if(f&AlignedBit) res += " | Aligned"; + if(f&NestByRefBit) res += " | NestByRef"; + return res; +} template<typename Dst, typename Src> bool test_assign(const Dst&, const Src&, int traversal, int unrolling) { - internal::assign_traits<Dst,Src>::debug(); - bool res = internal::assign_traits<Dst,Src>::Traversal==traversal - && internal::assign_traits<Dst,Src>::Unrolling==unrolling; +#ifdef EIGEN_TEST_EVALUATORS + typedef internal::copy_using_evaluator_traits<internal::evaluator<Dst>,internal::evaluator<Src>, internal::assign_op<typename Dst::Scalar> > traits; +#else + typedef internal::assign_traits<Dst,Src> traits; +#endif + bool res = traits::Traversal==traversal && traits::Unrolling==unrolling; if(!res) { + std::cerr << "Src: " << demangle_flags(Src::Flags) << std::endl; +#ifdef EIGEN_TEST_EVALUATORS + std::cerr << " " << demangle_flags(internal::evaluator<Src>::Flags) << std::endl; +#endif + std::cerr << "Dst: " << demangle_flags(Dst::Flags) << std::endl; +#ifdef EIGEN_TEST_EVALUATORS + std::cerr << " " << demangle_flags(internal::evaluator<Dst>::Flags) << std::endl; +#endif + traits::debug(); std::cerr << " Expected Traversal == " << demangle_traversal(traversal) - << " got " << demangle_traversal(internal::assign_traits<Dst,Src>::Traversal) << "\n"; + << " got " << demangle_traversal(traits::Traversal) << "\n"; std::cerr << " Expected Unrolling == " << demangle_unrolling(unrolling) - << " got " << demangle_unrolling(internal::assign_traits<Dst,Src>::Unrolling) << "\n"; + << " got " << demangle_unrolling(traits::Unrolling) << "\n"; } return res; } @@ -47,15 +71,27 @@ bool test_assign(const Dst&, const Src&, int traversal, int unrolling) template<typename Dst, typename Src> bool test_assign(int traversal, int unrolling) { - internal::assign_traits<Dst,Src>::debug(); - bool res = internal::assign_traits<Dst,Src>::Traversal==traversal - && internal::assign_traits<Dst,Src>::Unrolling==unrolling; +#ifdef EIGEN_TEST_EVALUATORS + typedef internal::copy_using_evaluator_traits<internal::evaluator<Dst>,internal::evaluator<Src>, internal::assign_op<typename Dst::Scalar> > traits; +#else + typedef internal::assign_traits<Dst,Src> traits; +#endif + bool res = traits::Traversal==traversal && traits::Unrolling==unrolling; if(!res) { + std::cerr << "Src: " << demangle_flags(Src::Flags) << std::endl; +#ifdef EIGEN_TEST_EVALUATORS + std::cerr << " " << demangle_flags(internal::evaluator<Src>::Flags) << std::endl; +#endif + std::cerr << "Dst: " << demangle_flags(Dst::Flags) << std::endl; +#ifdef EIGEN_TEST_EVALUATORS + std::cerr << " " << demangle_flags(internal::evaluator<Dst>::Flags) << std::endl; +#endif + traits::debug(); std::cerr << " Expected Traversal == " << demangle_traversal(traversal) - << " got " << demangle_traversal(internal::assign_traits<Dst,Src>::Traversal) << "\n"; + << " got " << demangle_traversal(traits::Traversal) << "\n"; std::cerr << " Expected Unrolling == " << demangle_unrolling(unrolling) - << " got " << demangle_unrolling(internal::assign_traits<Dst,Src>::Unrolling) << "\n"; + << " got " << demangle_unrolling(traits::Unrolling) << "\n"; } return res; } @@ -63,10 +99,21 @@ bool test_assign(int traversal, int unrolling) template<typename Xpr> bool test_redux(const Xpr&, int traversal, int unrolling) { +#ifdef EIGEN_TEST_EVALUATORS + typedef internal::redux_traits<internal::scalar_sum_op<typename Xpr::Scalar>,internal::redux_evaluator<Xpr> > traits; +#else typedef internal::redux_traits<internal::scalar_sum_op<typename Xpr::Scalar>,Xpr> traits; +#endif + bool res = traits::Traversal==traversal && traits::Unrolling==unrolling; if(!res) { + std::cerr << demangle_flags(Xpr::Flags) << std::endl; +#ifdef EIGEN_TEST_EVALUATORS + std::cerr << demangle_flags(internal::evaluator<Xpr>::Flags) << std::endl; +#endif + traits::debug(); + std::cerr << " Expected Traversal == " << demangle_traversal(traversal) << " got " << demangle_traversal(traits::Traversal) << "\n"; std::cerr << " Expected Unrolling == " << demangle_unrolling(unrolling) diff --git a/test/vectorwiseop.cpp b/test/vectorwiseop.cpp index 6cd1acdda..1631d54c4 100644 --- a/test/vectorwiseop.cpp +++ b/test/vectorwiseop.cpp @@ -104,8 +104,8 @@ template<typename ArrayType> void vectorwiseop_array(const ArrayType& m) m2 = m1; // yes, there might be an aliasing issue there but ".rowwise() /=" - // is suppposed to evaluate " m2.colwise().sum()" into to temporary to avoid - // evaluating the reducions multiple times + // is supposed to evaluate " m2.colwise().sum()" into a temporary to avoid + // evaluating the reduction multiple times if(ArrayType::RowsAtCompileTime>2 || ArrayType::RowsAtCompileTime==Dynamic) { m2.rowwise() /= m2.colwise().sum(); diff --git a/unsupported/Eigen/MPRealSupport b/unsupported/Eigen/MPRealSupport index 632de3854..630e0aa77 100644 --- a/unsupported/Eigen/MPRealSupport +++ b/unsupported/Eigen/MPRealSupport @@ -159,10 +159,14 @@ int main() { if(rows==0 || cols==0 || depth==0) return; +<<<<<<< local + +======= +>>>>>>> other mpreal acc1(0,mpfr_get_prec(blockA[0].mpfr_srcptr())), tmp (0,mpfr_get_prec(blockA[0].mpfr_srcptr())); - + if(strideA==-1) strideA = depth; if(strideB==-1) strideB = depth; |