Evaluators: Make inner vectorization more similar to default traversal.

2 files changed, 36 insertions, 70 deletions

diff --git a/Eigen/src/Core/AssignEvaluator.h b/Eigen/src/Core/AssignEvaluator.h
index 0b90ef70f..78014c6f9 100644
--- a/Eigen/src/Core/AssignEvaluator.h
+++ b/Eigen/src/Core/AssignEvaluator.h
@@ -259,8 +259,11 @@ struct copy_using_evaluator_impl<DstXprType, SrcXprType, InnerVectorizedTraversa
     const Index outerSize = dst.outerSize();
     const Index packetSize = packet_traits<typename DstXprType::Scalar>::size;
     for(Index outer = 0; outer < outerSize; ++outer)
-      for(Index inner = 0; inner < innerSize; inner+=packetSize)
-	dstEvaluator.template writePacketByOuterInner<Aligned>(outer, inner, srcEvaluator.template packetByOuterInner<Aligned>(outer, inner));
+      for(Index inner = 0; inner < innerSize; inner+=packetSize) {
+	Index row = dst.rowIndexByOuterInner(outer, inner);
+	Index col = dst.colIndexByOuterInner(outer, inner);
+	dstEvaluator.template writePacket<Aligned>(row, col, srcEvaluator.template packet<Aligned>(row, col));
+      }
   }
 };
 
diff --git a/Eigen/src/Core/CoreEvaluators.h b/Eigen/src/Core/CoreEvaluators.h
index 734936681..008285b4c 100644
--- a/Eigen/src/Core/CoreEvaluators.h
+++ b/Eigen/src/Core/CoreEvaluators.h
@@ -75,30 +75,29 @@ struct evaluator_impl<Transpose<ExpressionType> >
     return m_argImpl.coeffRef(index);
   }
 
+  // TODO: Difference between PacketScalar and PacketReturnType?
   template<int LoadMode>
-  const typename ExpressionType::PacketScalar packet(Index index) const
+  const typename ExpressionType::PacketScalar packet(Index row, Index col) const
   {
-    return m_argImpl.template packet<LoadMode>(index);
+    return m_argImpl.template packet<LoadMode>(col, row);
   }
 
-  // TODO: Difference between PacketScalar and PacketReturnType?
-  // TODO: Get this function by inheriting from DenseCoeffBase?
-  template<int LoadMode>  
-  const typename ExpressionType::PacketScalar packetByOuterInner(Index outer, Index inner) const
+  template<int LoadMode>
+  const typename ExpressionType::PacketScalar packet(Index index) const
   {
-    return m_argImpl.template packetByOuterInner<LoadMode>(outer, inner);
+    return m_argImpl.template packet<LoadMode>(index);
   }
 
   template<int StoreMode> 
-  void writePacket(Index index, const typename ExpressionType::PacketScalar& x)
+  void writePacket(Index row, Index col, const typename ExpressionType::PacketScalar& x)
   {
-    m_argImpl.template writePacket<StoreMode>(index, x);
+    m_argImpl.template writePacket<StoreMode>(col, row, x);
   }
 
   template<int StoreMode> 
-  void writePacketByOuterInner(Index outer, Index inner, const typename ExpressionType::PacketScalar& x)
+  void writePacket(Index index, const typename ExpressionType::PacketScalar& x)
   {
-    m_argImpl.template writePacketByOuterInner<StoreMode>(outer, inner, x);
+    m_argImpl.template writePacket<StoreMode>(index, x);
   }
 
 protected:
@@ -116,16 +115,6 @@ struct evaluator_impl<Matrix<Scalar, Rows, Cols, Options, MaxRows, MaxCols> >
 
   typedef typename MatrixType::Index Index;
 
-  Index colIndexByOuterInner(Index outer, Index inner) const 
-  { 
-    return m_matrix.colIndexByOuterInner(outer, inner); 
-  }
-
-  Index rowIndexByOuterInner(Index outer, Index inner) const 
-  { 
-    return m_matrix.rowIndexByOuterInner(outer, inner); 
-  }
-
   typename MatrixType::CoeffReturnType coeff(Index i, Index j) const
   {
     return m_matrix.coeff(i, j);
@@ -147,35 +136,29 @@ struct evaluator_impl<Matrix<Scalar, Rows, Cols, Options, MaxRows, MaxCols> >
   }
 
   template<int LoadMode> 
-  typename MatrixType::PacketReturnType packet(Index index) const
-  {
-    // eigen_internal_assert(index >= 0 && index < size());
-    return m_matrix.template packet<LoadMode>(index);
-  }
-
-  template<int LoadMode> 
   typename MatrixType::PacketReturnType packet(Index row, Index col) const
   {
     return m_matrix.template packet<LoadMode>(row, col);
   }
 
   template<int LoadMode> 
-  typename MatrixType::PacketReturnType packetByOuterInner(Index outer, Index inner) const
+  typename MatrixType::PacketReturnType packet(Index index) const
   {
-    return m_matrix.template packetByOuterInner<LoadMode>(outer, inner);
+    // eigen_internal_assert(index >= 0 && index < size());
+    return m_matrix.template packet<LoadMode>(index);
   }
 
   template<int StoreMode> 
-  void writePacket(Index index, const typename MatrixType::PacketScalar& x)
+  void writePacket(Index row, Index col, const typename MatrixType::PacketScalar& x)
   {
-    // eigen_internal_assert(index >= 0 && index < size());
-    m_matrix.const_cast_derived().template writePacket<StoreMode>(index, x);
+    m_matrix.const_cast_derived().template writePacket<StoreMode>(row, col, x);
   }
 
   template<int StoreMode> 
-  void writePacketByOuterInner(Index outer, Index inner, const typename MatrixType::PacketScalar& x)
+  void writePacket(Index index, const typename MatrixType::PacketScalar& x)
   {
-    m_matrix.const_cast_derived().template writePacketByOuterInner<StoreMode>(outer, inner, x);
+    // eigen_internal_assert(index >= 0 && index < size());
+    m_matrix.const_cast_derived().template writePacket<StoreMode>(index, x);
   }
 
 protected:
@@ -195,16 +178,16 @@ struct evaluator_impl<Array<Scalar, Rows, Cols, Options, MaxRows, MaxCols> >
 
   typedef typename ArrayType::Index Index;
   
-  Index colIndexByOuterInner(Index outer, Index inner) const 
-  { 
-    return m_array.colIndexByOuterInner(outer, inner); 
-  }
-
   typename ArrayType::CoeffReturnType coeff(Index i, Index j) const
   {
     return m_array.coeff(i, j);
   }
 
+  typename ArrayType::CoeffReturnType coeff(Index index) const
+  {
+    return m_array.coeff(index);
+  }
+
   typename ArrayType::Scalar& coeffRef(Index i, Index j)
   {
     return m_array.const_cast_derived().coeffRef(i, j);
@@ -216,35 +199,29 @@ struct evaluator_impl<Array<Scalar, Rows, Cols, Options, MaxRows, MaxCols> >
   }
 
   template<int LoadMode> 
-  typename ArrayType::PacketReturnType packet(Index index) const
-  {
-    // eigen_internal_assert(index >= 0 && index < size());
-    return m_array.template packet<LoadMode>(index);
-  }
-
-  template<int LoadMode> 
   typename ArrayType::PacketReturnType packet(Index row, Index col) const
   {
     return m_array.template packet<LoadMode>(row, col);
   }
 
   template<int LoadMode> 
-  typename ArrayType::PacketReturnType packetByOuterInner(Index outer, Index inner) const
+  typename ArrayType::PacketReturnType packet(Index index) const
   {
-    return m_array.template packetByOuterInner<LoadMode>(outer, inner);
+    // eigen_internal_assert(index >= 0 && index < size());
+    return m_array.template packet<LoadMode>(index);
   }
 
   template<int StoreMode> 
-  void writePacket(Index index, const typename ArrayType::PacketScalar& x)
+  void writePacket(Index row, Index col, const typename ArrayType::PacketScalar& x)
   {
-    // eigen_internal_assert(index >= 0 && index < size());
-    m_array.const_cast_derived().template writePacket<StoreMode>(index, x);
+    m_array.const_cast_derived().template writePacket<StoreMode>(row, col, x);
   }
 
   template<int StoreMode> 
-  void writePacketByOuterInner(Index outer, Index inner, const typename ArrayType::PacketScalar& x)
+  void writePacket(Index index, const typename ArrayType::PacketScalar& x)
   {
-    m_array.const_cast_derived().template writePacketByOuterInner<StoreMode>(outer, inner, x);
+    // eigen_internal_assert(index >= 0 && index < size());
+    m_array.const_cast_derived().template writePacket<StoreMode>(index, x);
   }
 
 protected:
@@ -315,13 +292,6 @@ struct evaluator_impl<CwiseUnaryOp<UnaryOp, ArgType> >
     return m_unaryOp.functor().packetOp(m_argImpl.template packet<LoadMode>(row, col));
   }
 
-  template<int LoadMode>
-  typename UnaryOpType::PacketScalar packetByOuterInner(Index outer, Index inner) const
-  {
-    return packet<LoadMode>(m_argImpl.rowIndexByOuterInner(outer, inner),
-			    m_argImpl.colIndexByOuterInner(outer, inner));
-  }
-
 protected:
   const UnaryOpType& m_unaryOp;
   typename evaluator<ArgType>::type m_argImpl;
@@ -362,13 +332,6 @@ struct evaluator_impl<CwiseBinaryOp<BinaryOp, Lhs, Rhs> >
 					 m_rhsImpl.template packet<LoadMode>(row, col));
   }
 
-  template<int LoadMode>
-  typename BinaryOpType::PacketScalar packetByOuterInner(Index outer, Index inner) const
-  {
-    return packet<LoadMode>(m_lhsImpl.rowIndexByOuterInner(outer, inner),
-			    m_lhsImpl.colIndexByOuterInner(outer, inner));
-  }
-
 protected:
   const BinaryOpType& m_binaryOp;
   typename evaluator<Lhs>::type m_lhsImpl;