aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Eigen/src/Core/arch/SSE/Complex.h48
-rw-r--r--Eigen/src/Core/arch/SSE/PacketMath.h1
-rw-r--r--Eigen/src/Core/products/GeneralBlockPanelKernel.h215
-rw-r--r--Eigen/src/Core/products/GeneralMatrixVector.h70
-rw-r--r--Eigen/src/Core/products/SelfadjointMatrixMatrix.h4
-rw-r--r--Eigen/src/Core/products/SelfadjointMatrixVector.h13
-rw-r--r--Eigen/src/Core/products/SelfadjointProduct.h15
-rw-r--r--Eigen/src/Core/products/TriangularMatrixMatrix.h6
-rw-r--r--Eigen/src/Core/products/TriangularSolverMatrix.h8
-rw-r--r--Eigen/src/Core/util/BlasUtil.h45
-rw-r--r--Eigen/src/Core/util/ForwardDeclarations.h4
-rw-r--r--Eigen/src/Jacobi/Jacobi.h10
12 files changed, 246 insertions, 193 deletions
diff --git a/Eigen/src/Core/arch/SSE/Complex.h b/Eigen/src/Core/arch/SSE/Complex.h
index 3f7a04b7d..fcc5219f5 100644
--- a/Eigen/src/Core/arch/SSE/Complex.h
+++ b/Eigen/src/Core/arch/SSE/Complex.h
@@ -79,9 +79,9 @@ template<> EIGEN_STRONG_INLINE Packet2cf ei_pmul<Packet2cf>(const Packet2cf& a,
{
// TODO optimize it for SSE3 and 4
const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x80000000,0x00000000,0x80000000,0x00000000));
- return Packet2cf(_mm_add_ps(_mm_mul_ps(_mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(a.v), 0xa0)), b.v),
- _mm_xor_ps(_mm_mul_ps(_mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(a.v), 0xf5)),
- _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(b.v), 0xb1 ))), mask)));
+ return Packet2cf(_mm_add_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v),
+ _mm_xor_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 1, 1, 3, 3),
+ ei_vec4f_swizzle1(b.v, 1, 0, 3, 2)), mask)));
}
// template<> EIGEN_STRONG_INLINE Packet2cf ei_pmadd<Packet2cf>(const Packet2cf& a, const Packet2cf& b, const Packet2cf& c)
@@ -157,4 +157,46 @@ struct ei_palign_impl<Offset,Packet2cf>
}
};
+template<> struct ei_conj_helper<Packet2cf, Packet2cf, false,true>
+{
+ EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, const Packet2cf& c) const
+ { return ei_padd(pmul(x,y),c); }
+
+ EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const
+ {
+ const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000));
+ return Packet2cf(_mm_add_ps(_mm_xor_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v), mask),
+ _mm_mul_ps(ei_vec4f_swizzle1(a.v, 1, 1, 3, 3),
+ ei_vec4f_swizzle1(b.v, 1, 0, 3, 2))));
+ }
+};
+
+template<> struct ei_conj_helper<Packet2cf, Packet2cf, true,false>
+{
+ EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, const Packet2cf& c) const
+ { return ei_padd(pmul(x,y),c); }
+
+ EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const
+ {
+ const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000));
+ return Packet2cf(_mm_add_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v),
+ _mm_xor_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 1, 1, 3, 3),
+ ei_vec4f_swizzle1(b.v, 1, 0, 3, 2)), mask)));
+ }
+};
+
+template<> struct ei_conj_helper<Packet2cf, Packet2cf, true,true>
+{
+ EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& x, const Packet2cf& y, const Packet2cf& c) const
+ { return ei_padd(pmul(x,y),c); }
+
+ EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) const
+ {
+ const __m128 mask = _mm_castsi128_ps(_mm_setr_epi32(0x00000000,0x80000000,0x00000000,0x80000000));
+ return Packet2cf(_mm_sub_ps(_mm_xor_ps(_mm_mul_ps(ei_vec4f_swizzle1(a.v, 0, 0, 2, 2), b.v), mask),
+ _mm_mul_ps(ei_vec4f_swizzle1(a.v, 1, 1, 3, 3),
+ ei_vec4f_swizzle1(b.v, 1, 0, 3, 2))));
+ }
+};
+
#endif // EIGEN_COMPLEX_SSE_H
diff --git a/Eigen/src/Core/arch/SSE/PacketMath.h b/Eigen/src/Core/arch/SSE/PacketMath.h
index 8a3cdf679..141754d98 100644
--- a/Eigen/src/Core/arch/SSE/PacketMath.h
+++ b/Eigen/src/Core/arch/SSE/PacketMath.h
@@ -58,6 +58,7 @@ template<> struct ei_is_arithmetic<__m128d> { enum { ret = true }; };
#define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
const Packet4i ei_p4i_##NAME = ei_pset1<int>(X)
+
template<> struct ei_packet_traits<float> : ei_default_packet_traits
{
typedef Packet4f type; enum {size=4};
diff --git a/Eigen/src/Core/products/GeneralBlockPanelKernel.h b/Eigen/src/Core/products/GeneralBlockPanelKernel.h
index 4a2ebb713..cf133f68f 100644
--- a/Eigen/src/Core/products/GeneralBlockPanelKernel.h
+++ b/Eigen/src/Core/products/GeneralBlockPanelKernel.h
@@ -134,13 +134,13 @@ inline void computeProductBlockingSizes(std::ptrdiff_t& k, std::ptrdiff_t& m, st
}
#ifdef EIGEN_HAS_FUSE_CJMADD
- #define CJMADD(A,B,C,T) C = cj.pmadd(A,B,C);
+ #define MADD(CJ,A,B,C,T) C = CJ.pmadd(A,B,C);
#else
- #define CJMADD(A,B,C,T) T = B; T = cj.pmul(A,T); C = ei_padd(C,T);
+ #define MADD(CJ,A,B,C,T) T = B; T = CJ.pmul(A,T); C = ei_padd(C,T);
#endif
// optimized GEneral packed Block * packed Panel product kernel
-template<typename Scalar, typename Index, int mr, int nr, typename Conj>
+template<typename Scalar, typename Index, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
struct ei_gebp_kernel
{
void operator()(Scalar* res, Index resStride, const Scalar* blockA, const Scalar* blockB, Index rows, Index depth, Index cols,
@@ -150,7 +150,8 @@ struct ei_gebp_kernel
enum { PacketSize = ei_packet_traits<Scalar>::size };
if(strideA==-1) strideA = depth;
if(strideB==-1) strideB = depth;
- Conj cj;
+ ei_conj_helper<Scalar,Scalar,ConjugateLhs,ConjugateRhs> cj;
+ ei_conj_helper<PacketType,PacketType,ConjugateLhs,ConjugateRhs> pcj;
Index packet_cols = (cols/nr) * nr;
const Index peeled_mc = (rows/mr)*mr;
const Index peeled_mc2 = peeled_mc + (rows-peeled_mc >= PacketSize ? PacketSize : 0);
@@ -263,38 +264,38 @@ EIGEN_ASM_COMMENT("mybegin");
A0 = ei_pload(&blA[0*PacketSize]);
A1 = ei_pload(&blA[1*PacketSize]);
B0 = ei_pload(&blB[0*PacketSize]);
- CJMADD(A0,B0,C0,T0);
- CJMADD(A1,B0,C4,B0);
+ MADD(pcj,A0,B0,C0,T0);
+ MADD(pcj,A1,B0,C4,B0);
B0 = ei_pload(&blB[1*PacketSize]);
- CJMADD(A0,B0,C1,T0);
- CJMADD(A1,B0,C5,B0);
+ MADD(pcj,A0,B0,C1,T0);
+ MADD(pcj,A1,B0,C5,B0);
A0 = ei_pload(&blA[2*PacketSize]);
A1 = ei_pload(&blA[3*PacketSize]);
B0 = ei_pload(&blB[2*PacketSize]);
- CJMADD(A0,B0,C0,T0);
- CJMADD(A1,B0,C4,B0);
+ MADD(pcj,A0,B0,C0,T0);
+ MADD(pcj,A1,B0,C4,B0);
B0 = ei_pload(&blB[3*PacketSize]);
- CJMADD(A0,B0,C1,T0);
- CJMADD(A1,B0,C5,B0);
+ MADD(pcj,A0,B0,C1,T0);
+ MADD(pcj,A1,B0,C5,B0);
A0 = ei_pload(&blA[4*PacketSize]);
A1 = ei_pload(&blA[5*PacketSize]);
B0 = ei_pload(&blB[4*PacketSize]);
- CJMADD(A0,B0,C0,T0);
- CJMADD(A1,B0,C4,B0);
+ MADD(pcj,A0,B0,C0,T0);
+ MADD(pcj,A1,B0,C4,B0);
B0 = ei_pload(&blB[5*PacketSize]);
- CJMADD(A0,B0,C1,T0);
- CJMADD(A1,B0,C5,B0);
+ MADD(pcj,A0,B0,C1,T0);
+ MADD(pcj,A1,B0,C5,B0);
A0 = ei_pload(&blA[6*PacketSize]);
A1 = ei_pload(&blA[7*PacketSize]);
B0 = ei_pload(&blB[6*PacketSize]);
- CJMADD(A0,B0,C0,T0);
- CJMADD(A1,B0,C4,B0);
+ MADD(pcj,A0,B0,C0,T0);
+ MADD(pcj,A1,B0,C4,B0);
B0 = ei_pload(&blB[7*PacketSize]);
- CJMADD(A0,B0,C1,T0);
- CJMADD(A1,B0,C5,B0);
+ MADD(pcj,A0,B0,C1,T0);
+ MADD(pcj,A1,B0,C5,B0);
EIGEN_ASM_COMMENT("myend");
}
else
@@ -309,59 +310,59 @@ EIGEN_ASM_COMMENT("mybegin");
B0 = ei_pload(&blB[0*PacketSize]);
B1 = ei_pload(&blB[1*PacketSize]);
- CJMADD(A0,B0,C0,T0);
+ MADD(pcj,A0,B0,C0,T0);
B2 = ei_pload(&blB[2*PacketSize]);
- CJMADD(A1,B0,C4,B0);
+ MADD(pcj,A1,B0,C4,B0);
B3 = ei_pload(&blB[3*PacketSize]);
B0 = ei_pload(&blB[4*PacketSize]);
- CJMADD(A0,B1,C1,T0);
- CJMADD(A1,B1,C5,B1);
+ MADD(pcj,A0,B1,C1,T0);
+ MADD(pcj,A1,B1,C5,B1);
B1 = ei_pload(&blB[5*PacketSize]);
- CJMADD(A0,B2,C2,T0);
- CJMADD(A1,B2,C6,B2);
+ MADD(pcj,A0,B2,C2,T0);
+ MADD(pcj,A1,B2,C6,B2);
B2 = ei_pload(&blB[6*PacketSize]);
- CJMADD(A0,B3,C3,T0);
+ MADD(pcj,A0,B3,C3,T0);
A0 = ei_pload(&blA[2*PacketSize]);
- CJMADD(A1,B3,C7,B3);
+ MADD(pcj,A1,B3,C7,B3);
A1 = ei_pload(&blA[3*PacketSize]);
B3 = ei_pload(&blB[7*PacketSize]);
- CJMADD(A0,B0,C0,T0);
- CJMADD(A1,B0,C4,B0);
+ MADD(pcj,A0,B0,C0,T0);
+ MADD(pcj,A1,B0,C4,B0);
B0 = ei_pload(&blB[8*PacketSize]);
- CJMADD(A0,B1,C1,T0);
- CJMADD(A1,B1,C5,B1);
+ MADD(pcj,A0,B1,C1,T0);
+ MADD(pcj,A1,B1,C5,B1);
B1 = ei_pload(&blB[9*PacketSize]);
- CJMADD(A0,B2,C2,T0);
- CJMADD(A1,B2,C6,B2);
+ MADD(pcj,A0,B2,C2,T0);
+ MADD(pcj,A1,B2,C6,B2);
B2 = ei_pload(&blB[10*PacketSize]);
- CJMADD(A0,B3,C3,T0);
+ MADD(pcj,A0,B3,C3,T0);
A0 = ei_pload(&blA[4*PacketSize]);
- CJMADD(A1,B3,C7,B3);
+ MADD(pcj,A1,B3,C7,B3);
A1 = ei_pload(&blA[5*PacketSize]);
B3 = ei_pload(&blB[11*PacketSize]);
- CJMADD(A0,B0,C0,T0);
- CJMADD(A1,B0,C4,B0);
+ MADD(pcj,A0,B0,C0,T0);
+ MADD(pcj,A1,B0,C4,B0);
B0 = ei_pload(&blB[12*PacketSize]);
- CJMADD(A0,B1,C1,T0);
- CJMADD(A1,B1,C5,B1);
+ MADD(pcj,A0,B1,C1,T0);
+ MADD(pcj,A1,B1,C5,B1);
B1 = ei_pload(&blB[13*PacketSize]);
- CJMADD(A0,B2,C2,T0);
- CJMADD(A1,B2,C6,B2);
+ MADD(pcj,A0,B2,C2,T0);
+ MADD(pcj,A1,B2,C6,B2);
B2 = ei_pload(&blB[14*PacketSize]);
- CJMADD(A0,B3,C3,T0);
+ MADD(pcj,A0,B3,C3,T0);
A0 = ei_pload(&blA[6*PacketSize]);
- CJMADD(A1,B3,C7,B3);
+ MADD(pcj,A1,B3,C7,B3);
A1 = ei_pload(&blA[7*PacketSize]);
B3 = ei_pload(&blB[15*PacketSize]);
- CJMADD(A0,B0,C0,T0);
- CJMADD(A1,B0,C4,B0);
- CJMADD(A0,B1,C1,T0);
- CJMADD(A1,B1,C5,B1);
- CJMADD(A0,B2,C2,T0);
- CJMADD(A1,B2,C6,B2);
- CJMADD(A0,B3,C3,T0);
- CJMADD(A1,B3,C7,B3);
+ MADD(pcj,A0,B0,C0,T0);
+ MADD(pcj,A1,B0,C4,B0);
+ MADD(pcj,A0,B1,C1,T0);
+ MADD(pcj,A1,B1,C5,B1);
+ MADD(pcj,A0,B2,C2,T0);
+ MADD(pcj,A1,B2,C6,B2);
+ MADD(pcj,A0,B3,C3,T0);
+ MADD(pcj,A1,B3,C7,B3);
EIGEN_ASM_COMMENT("myend");
}
@@ -381,11 +382,11 @@ EIGEN_ASM_COMMENT("myend");
A0 = ei_pload(&blA[0*PacketSize]);
A1 = ei_pload(&blA[1*PacketSize]);
B0 = ei_pload(&blB[0*PacketSize]);
- CJMADD(A0,B0,C0,T0);
- CJMADD(A1,B0,C4,B0);
+ MADD(pcj,A0,B0,C0,T0);
+ MADD(pcj,A1,B0,C4,B0);
B0 = ei_pload(&blB[1*PacketSize]);
- CJMADD(A0,B0,C1,T0);
- CJMADD(A1,B0,C5,B0);
+ MADD(pcj,A0,B0,C1,T0);
+ MADD(pcj,A1,B0,C5,B0);
}
else
{
@@ -399,16 +400,16 @@ EIGEN_ASM_COMMENT("myend");
B0 = ei_pload(&blB[0*PacketSize]);
B1 = ei_pload(&blB[1*PacketSize]);
- CJMADD(A0,B0,C0,T0);
+ MADD(pcj,A0,B0,C0,T0);
B2 = ei_pload(&blB[2*PacketSize]);
- CJMADD(A1,B0,C4,B0);
+ MADD(pcj,A1,B0,C4,B0);
B3 = ei_pload(&blB[3*PacketSize]);
- CJMADD(A0,B1,C1,T0);
- CJMADD(A1,B1,C5,B1);
- CJMADD(A0,B2,C2,T0);
- CJMADD(A1,B2,C6,B2);
- CJMADD(A0,B3,C3,T0);
- CJMADD(A1,B3,C7,B3);
+ MADD(pcj,A0,B1,C1,T0);
+ MADD(pcj,A1,B1,C5,B1);
+ MADD(pcj,A0,B2,C2,T0);
+ MADD(pcj,A1,B2,C6,B2);
+ MADD(pcj,A0,B3,C3,T0);
+ MADD(pcj,A1,B3,C7,B3);
}
blB += nr*PacketSize;
@@ -468,23 +469,23 @@ EIGEN_ASM_COMMENT("myend");
A0 = ei_pload(&blA[0*PacketSize]);
B0 = ei_pload(&blB[0*PacketSize]);
B1 = ei_pload(&blB[1*PacketSize]);
- CJMADD(A0,B0,C0,B0);
+ MADD(pcj,A0,B0,C0,B0);
B0 = ei_pload(&blB[2*PacketSize]);
- CJMADD(A0,B1,C1,B1);
+ MADD(pcj,A0,B1,C1,B1);
A0 = ei_pload(&blA[1*PacketSize]);
B1 = ei_pload(&blB[3*PacketSize]);
- CJMADD(A0,B0,C0,B0);
+ MADD(pcj,A0,B0,C0,B0);
B0 = ei_pload(&blB[4*PacketSize]);
- CJMADD(A0,B1,C1,B1);
+ MADD(pcj,A0,B1,C1,B1);
A0 = ei_pload(&blA[2*PacketSize]);
B1 = ei_pload(&blB[5*PacketSize]);
- CJMADD(A0,B0,C0,B0);
+ MADD(pcj,A0,B0,C0,B0);
B0 = ei_pload(&blB[6*PacketSize]);
- CJMADD(A0,B1,C1,B1);
+ MADD(pcj,A0,B1,C1,B1);
A0 = ei_pload(&blA[3*PacketSize]);
B1 = ei_pload(&blB[7*PacketSize]);
- CJMADD(A0,B0,C0,B0);
- CJMADD(A0,B1,C1,B1);
+ MADD(pcj,A0,B0,C0,B0);
+ MADD(pcj,A0,B1,C1,B1);
}
else
{
@@ -494,41 +495,41 @@ EIGEN_ASM_COMMENT("myend");
B0 = ei_pload(&blB[0*PacketSize]);
B1 = ei_pload(&blB[1*PacketSize]);
- CJMADD(A0,B0,C0,B0);
+ MADD(pcj,A0,B0,C0,B0);
B2 = ei_pload(&blB[2*PacketSize]);
B3 = ei_pload(&blB[3*PacketSize]);
B0 = ei_pload(&blB[4*PacketSize]);
- CJMADD(A0,B1,C1,B1);
+ MADD(pcj,A0,B1,C1,B1);
B1 = ei_pload(&blB[5*PacketSize]);
- CJMADD(A0,B2,C2,B2);
+ MADD(pcj,A0,B2,C2,B2);
B2 = ei_pload(&blB[6*PacketSize]);
- CJMADD(A0,B3,C3,B3);
+ MADD(pcj,A0,B3,C3,B3);
A0 = ei_pload(&blA[1*PacketSize]);
B3 = ei_pload(&blB[7*PacketSize]);
- CJMADD(A0,B0,C0,B0);
+ MADD(pcj,A0,B0,C0,B0);
B0 = ei_pload(&blB[8*PacketSize]);
- CJMADD(A0,B1,C1,B1);
+ MADD(pcj,A0,B1,C1,B1);
B1 = ei_pload(&blB[9*PacketSize]);
- CJMADD(A0,B2,C2,B2);
+ MADD(pcj,A0,B2,C2,B2);
B2 = ei_pload(&blB[10*PacketSize]);
- CJMADD(A0,B3,C3,B3);
+ MADD(pcj,A0,B3,C3,B3);
A0 = ei_pload(&blA[2*PacketSize]);
B3 = ei_pload(&blB[11*PacketSize]);
- CJMADD(A0,B0,C0,B0);
+ MADD(pcj,A0,B0,C0,B0);
B0 = ei_pload(&blB[12*PacketSize]);
- CJMADD(A0,B1,C1,B1);
+ MADD(pcj,A0,B1,C1,B1);
B1 = ei_pload(&blB[13*PacketSize]);
- CJMADD(A0,B2,C2,B2);
+ MADD(pcj,A0,B2,C2,B2);
B2 = ei_pload(&blB[14*PacketSize]);
- CJMADD(A0,B3,C3,B3);
+ MADD(pcj,A0,B3,C3,B3);
A0 = ei_pload(&blA[3*PacketSize]);
B3 = ei_pload(&blB[15*PacketSize]);
- CJMADD(A0,B0,C0,B0);
- CJMADD(A0,B1,C1,B1);
- CJMADD(A0,B2,C2,B2);
- CJMADD(A0,B3,C3,B3);
+ MADD(pcj,A0,B0,C0,B0);
+ MADD(pcj,A0,B1,C1,B1);
+ MADD(pcj,A0,B2,C2,B2);
+ MADD(pcj,A0,B3,C3,B3);
}
blB += 4*nr*PacketSize;
@@ -546,9 +547,9 @@ EIGEN_ASM_COMMENT("myend");
A0 = ei_pload(&blA[0*PacketSize]);
B0 = ei_pload(&blB[0*PacketSize]);
- CJMADD(A0,B0,C0,T0);
+ MADD(pcj,A0,B0,C0,T0);
B0 = ei_pload(&blB[1*PacketSize]);
- CJMADD(A0,B0,C1,T0);
+ MADD(pcj,A0,B0,C1,T0);
}
else
{
@@ -563,10 +564,10 @@ EIGEN_ASM_COMMENT("myend");
B2 = ei_pload(&blB[2*PacketSize]);
B3 = ei_pload(&blB[3*PacketSize]);
- CJMADD(A0,B0,C0,T0);
- CJMADD(A0,B1,C1,T1);
- CJMADD(A0,B2,C2,T0);
- CJMADD(A0,B3,C3,T1);
+ MADD(pcj,A0,B0,C0,T0);
+ MADD(pcj,A0,B1,C1,T1);
+ MADD(pcj,A0,B2,C2,T0);
+ MADD(pcj,A0,B3,C3,T1);
}
blB += nr*PacketSize;
@@ -598,9 +599,9 @@ EIGEN_ASM_COMMENT("myend");
A0 = blA[k];
B0 = blB[0*PacketSize];
- CJMADD(A0,B0,C0,T0);
+ MADD(cj,A0,B0,C0,T0);
B0 = blB[1*PacketSize];
- CJMADD(A0,B0,C1,T0);
+ MADD(cj,A0,B0,C1,T0);
}
else
{
@@ -615,10 +616,10 @@ EIGEN_ASM_COMMENT("myend");
B2 = blB[2*PacketSize];
B3 = blB[3*PacketSize];
- CJMADD(A0,B0,C0,T0);
- CJMADD(A0,B1,C1,T1);
- CJMADD(A0,B2,C2,T0);
- CJMADD(A0,B3,C3,T1);
+ MADD(cj,A0,B0,C0,T0);
+ MADD(cj,A0,B1,C1,T1);
+ MADD(cj,A0,B2,C2,T0);
+ MADD(cj,A0,B3,C3,T1);
}
blB += nr*PacketSize;
@@ -664,8 +665,8 @@ EIGEN_ASM_COMMENT("myend");
A0 = ei_pload(&blA[0*PacketSize]);
A1 = ei_pload(&blA[1*PacketSize]);
B0 = ei_pload(&blB[0*PacketSize]);
- CJMADD(A0,B0,C0,T0);
- CJMADD(A1,B0,C4,T1);
+ MADD(pcj,A0,B0,C0,T0);
+ MADD(pcj,A1,B0,C4,T1);
blB += PacketSize;
blA += mr;
@@ -686,8 +687,7 @@ EIGEN_ASM_COMMENT("myend");
for(Index k=0; k<depth; k++)
{
PacketType T0;
- CJMADD(ei_pload(blA), ei_pload(blB), C0, T0);
- //C0 = cj.pmadd(ei_pload(blA), ei_pload(blB), C0);
+ MADD(pcj,ei_pload(blA), ei_pload(blB), C0, T0);
blB += PacketSize;
blA += PacketSize;
}
@@ -704,7 +704,12 @@ EIGEN_ASM_COMMENT("myend");
// FIXME directly use blockB ??
const Scalar* blB = unpackedB;
for(Index k=0; k<depth; k++)
- C0 = cj.pmadd(blA[k], blB[k*PacketSize], C0);
+ {
+ #ifndef EIGEN_HAS_FUSE_CJMADD
+ Scalar T0;
+ #endif
+ MADD(cj,blA[k], blB[k*PacketSize], C0, T0);
+ }
res[(j2+0)*resStride + i] += C0;
}
}
diff --git a/Eigen/src/Core/products/GeneralMatrixVector.h b/Eigen/src/Core/products/GeneralMatrixVector.h
index 5d8da247c..4cf843422 100644
--- a/Eigen/src/Core/products/GeneralMatrixVector.h
+++ b/Eigen/src/Core/products/GeneralMatrixVector.h
@@ -48,19 +48,20 @@ void ei_cache_friendly_product_colmajor_times_vector(
ei_pstore(&res[j], \
ei_padd(ei_pload(&res[j]), \
ei_padd( \
- ei_padd(cj.pmul(EIGEN_CAT(ei_ploa , A0)(&lhs0[j]), ptmp0), \
- cj.pmul(EIGEN_CAT(ei_ploa , A13)(&lhs1[j]), ptmp1)), \
- ei_padd(cj.pmul(EIGEN_CAT(ei_ploa , A2)(&lhs2[j]), ptmp2), \
- cj.pmul(EIGEN_CAT(ei_ploa , A13)(&lhs3[j]), ptmp3)) )))
-
- ei_conj_helper<ConjugateLhs,ConjugateRhs> cj;
- if(ConjugateRhs)
- alpha = ei_conj(alpha);
+ ei_padd(pcj.pmul(EIGEN_CAT(ei_ploa , A0)(&lhs0[j]), ptmp0), \
+ pcj.pmul(EIGEN_CAT(ei_ploa , A13)(&lhs1[j]), ptmp1)), \
+ ei_padd(pcj.pmul(EIGEN_CAT(ei_ploa , A2)(&lhs2[j]), ptmp2), \
+ pcj.pmul(EIGEN_CAT(ei_ploa , A13)(&lhs3[j]), ptmp3)) )))
typedef typename NumTraits<Scalar>::Real RealScalar;
typedef typename ei_packet_traits<Scalar>::type Packet;
const Index PacketSize = sizeof(Packet)/sizeof(Scalar);
+ ei_conj_helper<Scalar,Scalar,ConjugateLhs,ConjugateRhs> cj;
+ ei_conj_helper<Packet,Packet,ConjugateLhs,ConjugateRhs> pcj;
+ if(ConjugateRhs)
+ alpha = ei_conj(alpha);
+
enum { AllAligned = 0, EvenAligned, FirstAligned, NoneAligned };
const Index columnsAtOnce = 4;
const Index peels = 2;
@@ -168,19 +169,19 @@ void ei_cache_friendly_product_colmajor_times_vector(
A00 = ei_pload (&lhs0[j]);
A10 = ei_pload (&lhs0[j+PacketSize]);
- A00 = cj.pmadd(A00, ptmp0, ei_pload(&res[j]));
- A10 = cj.pmadd(A10, ptmp0, ei_pload(&res[j+PacketSize]));
+ A00 = pcj.pmadd(A00, ptmp0, ei_pload(&res[j]));
+ A10 = pcj.pmadd(A10, ptmp0, ei_pload(&res[j+PacketSize]));
- A00 = cj.pmadd(A01, ptmp1, A00);
+ A00 = pcj.pmadd(A01, ptmp1, A00);
A01 = ei_pload(&lhs1[j-1+2*PacketSize]); ei_palign<1>(A11,A01);
- A00 = cj.pmadd(A02, ptmp2, A00);
+ A00 = pcj.pmadd(A02, ptmp2, A00);
A02 = ei_pload(&lhs2[j-2+2*PacketSize]); ei_palign<2>(A12,A02);
- A00 = cj.pmadd(A03, ptmp3, A00);
+ A00 = pcj.pmadd(A03, ptmp3, A00);
ei_pstore(&res[j],A00);
A03 = ei_pload(&lhs3[j-3+2*PacketSize]); ei_palign<3>(A13,A03);
- A10 = cj.pmadd(A11, ptmp1, A10);
- A10 = cj.pmadd(A12, ptmp2, A10);
- A10 = cj.pmadd(A13, ptmp3, A10);
+ A10 = pcj.pmadd(A11, ptmp1, A10);
+ A10 = pcj.pmadd(A12, ptmp2, A10);
+ A10 = pcj.pmadd(A13, ptmp3, A10);
ei_pstore(&res[j+PacketSize],A10);
}
}
@@ -225,10 +226,10 @@ void ei_cache_friendly_product_colmajor_times_vector(
// process aligned result's coeffs
if ((size_t(lhs0+alignedStart)%sizeof(Packet))==0)
for (Index j = alignedStart;j<alignedSize;j+=PacketSize)
- ei_pstore(&res[j], cj.pmadd(ei_pload(&lhs0[j]), ptmp0, ei_pload(&res[j])));
+ ei_pstore(&res[j], pcj.pmadd(ei_pload(&lhs0[j]), ptmp0, ei_pload(&res[j])));
else
for (Index j = alignedStart;j<alignedSize;j+=PacketSize)
- ei_pstore(&res[j], cj.pmadd(ei_ploadu(&lhs0[j]), ptmp0, ei_pload(&res[j])));
+ ei_pstore(&res[j], pcj.pmadd(ei_ploadu(&lhs0[j]), ptmp0, ei_pload(&res[j])));
}
// process remaining scalars (or all if no explicit vectorization)
@@ -261,17 +262,18 @@ static EIGEN_DONT_INLINE void ei_cache_friendly_product_rowmajor_times_vector(
#define _EIGEN_ACCUMULATE_PACKETS(A0,A13,A2) {\
Packet b = ei_pload(&rhs[j]); \
- ptmp0 = cj.pmadd(EIGEN_CAT(ei_ploa,A0) (&lhs0[j]), b, ptmp0); \
- ptmp1 = cj.pmadd(EIGEN_CAT(ei_ploa,A13)(&lhs1[j]), b, ptmp1); \
- ptmp2 = cj.pmadd(EIGEN_CAT(ei_ploa,A2) (&lhs2[j]), b, ptmp2); \
- ptmp3 = cj.pmadd(EIGEN_CAT(ei_ploa,A13)(&lhs3[j]), b, ptmp3); }
-
- ei_conj_helper<ConjugateLhs,ConjugateRhs> cj;
+ ptmp0 = pcj.pmadd(EIGEN_CAT(ei_ploa,A0) (&lhs0[j]), b, ptmp0); \
+ ptmp1 = pcj.pmadd(EIGEN_CAT(ei_ploa,A13)(&lhs1[j]), b, ptmp1); \
+ ptmp2 = pcj.pmadd(EIGEN_CAT(ei_ploa,A2) (&lhs2[j]), b, ptmp2); \
+ ptmp3 = pcj.pmadd(EIGEN_CAT(ei_ploa,A13)(&lhs3[j]), b, ptmp3); }
typedef typename NumTraits<Scalar>::Real RealScalar;
typedef typename ei_packet_traits<Scalar>::type Packet;
const Index PacketSize = sizeof(Packet)/sizeof(Scalar);
+ ei_conj_helper<Scalar,Scalar,ConjugateLhs,ConjugateRhs> cj;
+ ei_conj_helper<Packet,Packet,ConjugateLhs,ConjugateRhs> pcj;
+
enum { AllAligned=0, EvenAligned=1, FirstAligned=2, NoneAligned=3 };
const Index rowsAtOnce = 4;
const Index peels = 2;
@@ -386,19 +388,19 @@ static EIGEN_DONT_INLINE void ei_cache_friendly_product_rowmajor_times_vector(
A12 = ei_pload(&lhs2[j-2+PacketSize]); ei_palign<2>(A02,A12);
A13 = ei_pload(&lhs3[j-3+PacketSize]); ei_palign<3>(A03,A13);
- ptmp0 = cj.pmadd(ei_pload (&lhs0[j]), b, ptmp0);
- ptmp1 = cj.pmadd(A01, b, ptmp1);
+ ptmp0 = pcj.pmadd(ei_pload (&lhs0[j]), b, ptmp0);
+ ptmp1 = pcj.pmadd(A01, b, ptmp1);
A01 = ei_pload(&lhs1[j-1+2*PacketSize]); ei_palign<1>(A11,A01);
- ptmp2 = cj.pmadd(A02, b, ptmp2);
+ ptmp2 = pcj.pmadd(A02, b, ptmp2);
A02 = ei_pload(&lhs2[j-2+2*PacketSize]); ei_palign<2>(A12,A02);
- ptmp3 = cj.pmadd(A03, b, ptmp3);
+ ptmp3 = pcj.pmadd(A03, b, ptmp3);
A03 = ei_pload(&lhs3[j-3+2*PacketSize]); ei_palign<3>(A13,A03);
b = ei_pload(&rhs[j+PacketSize]);
- ptmp0 = cj.pmadd(ei_pload (&lhs0[j+PacketSize]), b, ptmp0);
- ptmp1 = cj.pmadd(A11, b, ptmp1);
- ptmp2 = cj.pmadd(A12, b, ptmp2);
- ptmp3 = cj.pmadd(A13, b, ptmp3);
+ ptmp0 = pcj.pmadd(ei_pload (&lhs0[j+PacketSize]), b, ptmp0);
+ ptmp1 = pcj.pmadd(A11, b, ptmp1);
+ ptmp2 = pcj.pmadd(A12, b, ptmp2);
+ ptmp3 = pcj.pmadd(A13, b, ptmp3);
}
}
for (Index j = peeledSize; j<alignedSize; j+=PacketSize)
@@ -447,10 +449,10 @@ static EIGEN_DONT_INLINE void ei_cache_friendly_product_rowmajor_times_vector(
// process aligned rhs coeffs
if ((size_t(lhs0+alignedStart)%sizeof(Packet))==0)
for (Index j = alignedStart;j<alignedSize;j+=PacketSize)
- ptmp0 = cj.pmadd(ei_pload(&lhs0[j]), ei_pload(&rhs[j]), ptmp0);
+ ptmp0 = pcj.pmadd(ei_pload(&lhs0[j]), ei_pload(&rhs[j]), ptmp0);
else
for (Index j = alignedStart;j<alignedSize;j+=PacketSize)
- ptmp0 = cj.pmadd(ei_ploadu(&lhs0[j]), ei_pload(&rhs[j]), ptmp0);
+ ptmp0 = pcj.pmadd(ei_ploadu(&lhs0[j]), ei_pload(&rhs[j]), ptmp0);
tmp0 += ei_predux(ptmp0);
}
diff --git a/Eigen/src/Core/products/SelfadjointMatrixMatrix.h b/Eigen/src/Core/products/SelfadjointMatrixMatrix.h
index 24b711ced..d8fa1bd9c 100644
--- a/Eigen/src/Core/products/SelfadjointMatrixMatrix.h
+++ b/Eigen/src/Core/products/SelfadjointMatrixMatrix.h
@@ -270,7 +270,7 @@ struct ei_product_selfadjoint_matrix<Scalar,Index,LhsStorageOrder,true,Conjugate
Scalar* allocatedBlockB = ei_aligned_stack_new(Scalar, sizeB);
Scalar* blockB = allocatedBlockB + kc*Blocking::PacketSize*Blocking::nr;
- ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ei_conj_helper<ConjugateLhs,ConjugateRhs> > gebp_kernel;
+ ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
ei_symm_pack_lhs<Scalar, Index, Blocking::mr,LhsStorageOrder> pack_lhs;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder> pack_rhs;
ei_gemm_pack_lhs<Scalar, Index, Blocking::mr,LhsStorageOrder==RowMajor?ColMajor:RowMajor, true> pack_lhs_transposed;
@@ -353,7 +353,7 @@ struct ei_product_selfadjoint_matrix<Scalar,Index,LhsStorageOrder,false,Conjugat
Scalar* allocatedBlockB = ei_aligned_stack_new(Scalar, sizeB);
Scalar* blockB = allocatedBlockB + kc*Blocking::PacketSize*Blocking::nr;
- ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ei_conj_helper<ConjugateLhs,ConjugateRhs> > gebp_kernel;
+ ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
ei_gemm_pack_lhs<Scalar, Index, Blocking::mr,LhsStorageOrder> pack_lhs;
ei_symm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder> pack_rhs;
diff --git a/Eigen/src/Core/products/SelfadjointMatrixVector.h b/Eigen/src/Core/products/SelfadjointMatrixVector.h
index d6933adb6..4514c7692 100644
--- a/Eigen/src/Core/products/SelfadjointMatrixVector.h
+++ b/Eigen/src/Core/products/SelfadjointMatrixVector.h
@@ -46,8 +46,11 @@ static EIGEN_DONT_INLINE void ei_product_selfadjoint_vector(
FirstTriangular = IsRowMajor == IsLower
};
- ei_conj_helper<NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, IsRowMajor), ConjugateRhs> cj0;
- ei_conj_helper<NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, !IsRowMajor), ConjugateRhs> cj1;
+ ei_conj_helper<Scalar,Scalar,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, IsRowMajor), ConjugateRhs> cj0;
+ ei_conj_helper<Scalar,Scalar,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, !IsRowMajor), ConjugateRhs> cj1;
+
+ ei_conj_helper<Packet,Packet,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, IsRowMajor), ConjugateRhs> pcj0;
+ ei_conj_helper<Packet,Packet,NumTraits<Scalar>::IsComplex && EIGEN_LOGICAL_XOR(ConjugateLhs, !IsRowMajor), ConjugateRhs> pcj1;
Scalar cjAlpha = ConjugateRhs ? ei_conj(alpha) : alpha;
@@ -121,9 +124,9 @@ static EIGEN_DONT_INLINE void ei_product_selfadjoint_vector(
Packet Bi = ei_ploadu(rhsIt); rhsIt += PacketSize; // FIXME should be aligned in most cases
Packet Xi = ei_pload (resIt);
- Xi = cj0.pmadd(A0i,ptmp0, cj0.pmadd(A1i,ptmp1,Xi));
- ptmp2 = cj1.pmadd(A0i, Bi, ptmp2);
- ptmp3 = cj1.pmadd(A1i, Bi, ptmp3);
+ Xi = pcj0.pmadd(A0i,ptmp0, pcj0.pmadd(A1i,ptmp1,Xi));
+ ptmp2 = pcj1.pmadd(A0i, Bi, ptmp2);
+ ptmp3 = pcj1.pmadd(A1i, Bi, ptmp3);
ei_pstore(resIt,Xi); resIt += PacketSize;
}
for (size_t i=alignedEnd; i<endi; i++)
diff --git a/Eigen/src/Core/products/SelfadjointProduct.h b/Eigen/src/Core/products/SelfadjointProduct.h
index eaf634de3..40c0c9aac 100644
--- a/Eigen/src/Core/products/SelfadjointProduct.h
+++ b/Eigen/src/Core/products/SelfadjointProduct.h
@@ -32,7 +32,7 @@
**********************************************************************/
// forward declarations (defined at the end of this file)
-template<typename Scalar, typename Index, int mr, int nr, typename Conj, int UpLo>
+template<typename Scalar, typename Index, int mr, int nr, bool ConjLhs, bool ConjRhs, int UpLo>
struct ei_sybb_kernel;
/* Optimized selfadjoint product (_SYRK) */
@@ -84,12 +84,15 @@ struct ei_selfadjoint_product<Scalar, Index, MatStorageOrder, ColMajor, AAT, UpL
Scalar* blockB = allocatedBlockB + kc*Blocking::PacketSize*Blocking::nr;
// note that the actual rhs is the transpose/adjoint of mat
- typedef ei_conj_helper<NumTraits<Scalar>::IsComplex && !AAT, NumTraits<Scalar>::IsComplex && AAT> Conj;
+ enum {
+ ConjLhs = NumTraits<Scalar>::IsComplex && !AAT,
+ ConjRhs = NumTraits<Scalar>::IsComplex && AAT
+ };
- ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, Conj> gebp_kernel;
+ ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ConjLhs, ConjRhs> gebp_kernel;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,MatStorageOrder==RowMajor ? ColMajor : RowMajor> pack_rhs;
ei_gemm_pack_lhs<Scalar, Index, Blocking::mr,MatStorageOrder, false> pack_lhs;
- ei_sybb_kernel<Scalar, Index, Blocking::mr, Blocking::nr, Conj, UpLo> sybb;
+ ei_sybb_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ConjLhs, ConjRhs, UpLo> sybb;
for(Index k2=0; k2<depth; k2+=kc)
{
@@ -163,7 +166,7 @@ SelfAdjointView<MatrixType,UpLo>& SelfAdjointView<MatrixType,UpLo>
// while the selfadjoint block overlapping the diagonal is evaluated into a
// small temporary buffer which is then accumulated into the result using a
// triangular traversal.
-template<typename Scalar, typename Index, int mr, int nr, typename Conj, int UpLo>
+template<typename Scalar, typename Index, int mr, int nr, bool ConjLhs, bool ConjRhs, int UpLo>
struct ei_sybb_kernel
{
enum {
@@ -172,7 +175,7 @@ struct ei_sybb_kernel
};
void operator()(Scalar* res, Index resStride, const Scalar* blockA, const Scalar* blockB, Index size, Index depth, Scalar* workspace)
{
- ei_gebp_kernel<Scalar, Index, mr, nr, Conj> gebp_kernel;
+ ei_gebp_kernel<Scalar, Index, mr, nr, ConjLhs, ConjRhs> gebp_kernel;
Matrix<Scalar,BlockSize,BlockSize,ColMajor> buffer;
// let's process the block per panel of actual_mc x BlockSize,
diff --git a/Eigen/src/Core/products/TriangularMatrixMatrix.h b/Eigen/src/Core/products/TriangularMatrixMatrix.h
index fd0a7c2b2..be9362958 100644
--- a/Eigen/src/Core/products/TriangularMatrixMatrix.h
+++ b/Eigen/src/Core/products/TriangularMatrixMatrix.h
@@ -129,7 +129,7 @@ struct ei_product_triangular_matrix_matrix<Scalar,Index,Mode,true,
triangularBuffer.setZero();
triangularBuffer.diagonal().setOnes();
- ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ei_conj_helper<ConjugateLhs,ConjugateRhs> > gebp_kernel;
+ ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
ei_gemm_pack_lhs<Scalar, Index, Blocking::mr,LhsStorageOrder> pack_lhs;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder> pack_rhs;
@@ -254,10 +254,10 @@ struct ei_product_triangular_matrix_matrix<Scalar,Index,Mode,false,
triangularBuffer.setZero();
triangularBuffer.diagonal().setOnes();
- ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ei_conj_helper<ConjugateLhs,ConjugateRhs> > gebp_kernel;
+ ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
ei_gemm_pack_lhs<Scalar, Index, Blocking::mr,LhsStorageOrder> pack_lhs;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder> pack_rhs;
- ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder,true> pack_rhs_panel;
+ ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder,false,true> pack_rhs_panel;
for(Index k2=IsLower ? 0 : depth;
IsLower ? k2<depth : k2>0;
diff --git a/Eigen/src/Core/products/TriangularSolverMatrix.h b/Eigen/src/Core/products/TriangularSolverMatrix.h
index 7038627fb..0fce7159e 100644
--- a/Eigen/src/Core/products/TriangularSolverMatrix.h
+++ b/Eigen/src/Core/products/TriangularSolverMatrix.h
@@ -74,9 +74,9 @@ struct ei_triangular_solve_matrix<Scalar,Index,OnTheLeft,Mode,Conjugate,TriStora
Scalar* blockB = allocatedBlockB + kc*Blocking::PacketSize*Blocking::nr;
ei_conj_if<Conjugate> conj;
- ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ei_conj_helper<Conjugate,false> > gebp_kernel;
+ ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, Conjugate, false> gebp_kernel;
ei_gemm_pack_lhs<Scalar, Index, Blocking::mr,TriStorageOrder> pack_lhs;
- ei_gemm_pack_rhs<Scalar, Index, Blocking::nr, ColMajor, true> pack_rhs;
+ ei_gemm_pack_rhs<Scalar, Index, Blocking::nr, ColMajor, false, true> pack_rhs;
for(Index k2=IsLower ? 0 : size;
IsLower ? k2<size : k2>0;
@@ -212,9 +212,9 @@ struct ei_triangular_solve_matrix<Scalar,Index,OnTheRight,Mode,Conjugate,TriStor
Scalar* blockB = allocatedBlockB + kc*Blocking::PacketSize*Blocking::nr;
ei_conj_if<Conjugate> conj;
- ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, ei_conj_helper<false,Conjugate> > gebp_kernel;
+ ei_gebp_kernel<Scalar, Index, Blocking::mr, Blocking::nr, false, Conjugate> gebp_kernel;
ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder> pack_rhs;
- ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder,true> pack_rhs_panel;
+ ei_gemm_pack_rhs<Scalar, Index, Blocking::nr,RhsStorageOrder,false,true> pack_rhs_panel;
ei_gemm_pack_lhs<Scalar, Index, Blocking::mr, ColMajor, false, true> pack_lhs_panel;
for(Index k2=IsLower ? size : 0;
diff --git a/Eigen/src/Core/util/BlasUtil.h b/Eigen/src/Core/util/BlasUtil.h
index cd0ed0ede..38c86511c 100644
--- a/Eigen/src/Core/util/BlasUtil.h
+++ b/Eigen/src/Core/util/BlasUtil.h
@@ -29,12 +29,7 @@
// implement and control fast level 2 and level 3 BLAS-like routines.
// forward declarations
-
-// Provides scalar/packet-wise product and product with accumulation
-// with optional conjugation of the arguments.
-template<bool ConjLhs, bool ConjRhs> struct ei_conj_helper;
-
-template<typename Scalar, typename Index, int mr, int nr, typename Conj = ei_conj_helper<false,false> >
+template<typename Scalar, typename Index, int mr, int nr, bool ConjugateLhs=false, bool ConjugateRhs=false>
struct ei_gebp_kernel;
template<typename Scalar, typename Index, int nr, int StorageOrder, bool Conjugate = false, bool PanelMode=false>
@@ -58,42 +53,40 @@ template<bool ConjugateLhs, bool ConjugateRhs, typename Scalar, typename Index,
static void ei_cache_friendly_product_rowmajor_times_vector(
const Scalar* lhs, Index lhsStride, const Scalar* rhs, Index rhsSize, ResType& res, Scalar alpha);
-template<> struct ei_conj_helper<false,false>
+template<typename Scalar> struct ei_conj_helper<Scalar,Scalar,false,false>
{
- template<typename T>
- EIGEN_STRONG_INLINE T pmadd(const T& x, const T& y, const T& c) const { return ei_pmadd(x,y,c); }
- template<typename T>
- EIGEN_STRONG_INLINE T pmul(const T& x, const T& y) const { return ei_pmul(x,y); }
+ EIGEN_STRONG_INLINE Scalar pmadd(const Scalar& x, const Scalar& y, const Scalar& c) const { return ei_pmadd(x,y,c); }
+ EIGEN_STRONG_INLINE Scalar pmul(const Scalar& x, const Scalar& y) const { return ei_pmul(x,y); }
};
-template<> struct ei_conj_helper<false,true>
+template<typename RealScalar> struct ei_conj_helper<std::complex<RealScalar>, std::complex<RealScalar>, false,true>
{
- template<typename T> std::complex<T>
- pmadd(const std::complex<T>& x, const std::complex<T>& y, const std::complex<T>& c) const
+ typedef std::complex<RealScalar> Scalar;
+ EIGEN_STRONG_INLINE Scalar pmadd(const Scalar& x, const Scalar& y, const Scalar& c) const
{ return c + pmul(x,y); }
- template<typename T> std::complex<T> pmul(const std::complex<T>& x, const std::complex<T>& y) const
- { return std::complex<T>(ei_real(x)*ei_real(y) + ei_imag(x)*ei_imag(y), ei_imag(x)*ei_real(y) - ei_real(x)*ei_imag(y)); }
+ EIGEN_STRONG_INLINE Scalar pmul(const Scalar& x, const Scalar& y) const
+ { return Scalar(ei_real(x)*ei_real(y) + ei_imag(x)*ei_imag(y), ei_imag(x)*ei_real(y) - ei_real(x)*ei_imag(y)); }
};
-template<> struct ei_conj_helper<true,false>
+template<typename RealScalar> struct ei_conj_helper<std::complex<RealScalar>, std::complex<RealScalar>, true,false>
{
- template<typename T> std::complex<T>
- pmadd(const std::complex<T>& x, const std::complex<T>& y, const std::complex<T>& c) const
+ typedef std::complex<RealScalar> Scalar;
+ EIGEN_STRONG_INLINE Scalar pmadd(const Scalar& x, const Scalar& y, const Scalar& c) const
{ return c + pmul(x,y); }
- template<typename T> std::complex<T> pmul(const std::complex<T>& x, const std::complex<T>& y) const
- { return std::complex<T>(ei_real(x)*ei_real(y) + ei_imag(x)*ei_imag(y), ei_real(x)*ei_imag(y) - ei_imag(x)*ei_real(y)); }
+ EIGEN_STRONG_INLINE Scalar pmul(const Scalar& x, const Scalar& y) const
+ { return Scalar(ei_real(x)*ei_real(y) + ei_imag(x)*ei_imag(y), ei_real(x)*ei_imag(y) - ei_imag(x)*ei_real(y)); }
};
-template<> struct ei_conj_helper<true,true>
+template<typename RealScalar> struct ei_conj_helper<std::complex<RealScalar>, std::complex<RealScalar>, true,true>
{
- template<typename T> std::complex<T>
- pmadd(const std::complex<T>& x, const std::complex<T>& y, const std::complex<T>& c) const
+ typedef std::complex<RealScalar> Scalar;
+ EIGEN_STRONG_INLINE Scalar pmadd(const Scalar& x, const Scalar& y, const Scalar& c) const
{ return c + pmul(x,y); }
- template<typename T> std::complex<T> pmul(const std::complex<T>& x, const std::complex<T>& y) const
- { return std::complex<T>(ei_real(x)*ei_real(y) - ei_imag(x)*ei_imag(y), - ei_real(x)*ei_imag(y) - ei_imag(x)*ei_real(y)); }
+ EIGEN_STRONG_INLINE Scalar pmul(const Scalar& x, const Scalar& y) const
+ { return Scalar(ei_real(x)*ei_real(y) - ei_imag(x)*ei_imag(y), - ei_real(x)*ei_imag(y) - ei_imag(x)*ei_real(y)); }
};
// Lightweight helper class to access matrix coefficients.
diff --git a/Eigen/src/Core/util/ForwardDeclarations.h b/Eigen/src/Core/util/ForwardDeclarations.h
index ef923d867..a757bef76 100644
--- a/Eigen/src/Core/util/ForwardDeclarations.h
+++ b/Eigen/src/Core/util/ForwardDeclarations.h
@@ -106,6 +106,10 @@ template<typename Lhs, typename Rhs,
int ProductType = ei_product_type<Lhs,Rhs>::value>
struct ProductReturnType;
+// Provides scalar/packet-wise product and product with accumulation
+// with optional conjugation of the arguments.
+template<typename LhsScalar, typename RhsScalar, bool ConjLhs, bool ConjRhs> struct ei_conj_helper;
+
template<typename Scalar> struct ei_scalar_sum_op;
template<typename Scalar> struct ei_scalar_difference_op;
template<typename Scalar> struct ei_scalar_product_op;
diff --git a/Eigen/src/Jacobi/Jacobi.h b/Eigen/src/Jacobi/Jacobi.h
index 49a0d8f5d..94bb5569e 100644
--- a/Eigen/src/Jacobi/Jacobi.h
+++ b/Eigen/src/Jacobi/Jacobi.h
@@ -324,7 +324,7 @@ void /*EIGEN_DONT_INLINE*/ ei_apply_rotation_in_the_plane(VectorX& _x, VectorY&
const Packet pc = ei_pset1(Scalar(j.c()));
const Packet ps = ei_pset1(Scalar(j.s()));
- ei_conj_helper<NumTraits<Scalar>::IsComplex,false> cj;
+ ei_conj_helper<Packet,Packet,NumTraits<Scalar>::IsComplex,false> pcj;
for(Index i=0; i<alignedStart; ++i)
{
@@ -343,7 +343,7 @@ void /*EIGEN_DONT_INLINE*/ ei_apply_rotation_in_the_plane(VectorX& _x, VectorY&
{
Packet xi = ei_pload(px);
Packet yi = ei_pload(py);
- ei_pstore(px, ei_padd(ei_pmul(pc,xi),cj.pmul(ps,yi)));
+ ei_pstore(px, ei_padd(ei_pmul(pc,xi),pcj.pmul(ps,yi)));
ei_pstore(py, ei_psub(ei_pmul(pc,yi),ei_pmul(ps,xi)));
px += PacketSize;
py += PacketSize;
@@ -358,8 +358,8 @@ void /*EIGEN_DONT_INLINE*/ ei_apply_rotation_in_the_plane(VectorX& _x, VectorY&
Packet xi1 = ei_ploadu(px+PacketSize);
Packet yi = ei_pload (py);
Packet yi1 = ei_pload (py+PacketSize);
- ei_pstoreu(px, ei_padd(ei_pmul(pc,xi),cj.pmul(ps,yi)));
- ei_pstoreu(px+PacketSize, ei_padd(ei_pmul(pc,xi1),cj.pmul(ps,yi1)));
+ ei_pstoreu(px, ei_padd(ei_pmul(pc,xi),pcj.pmul(ps,yi)));
+ ei_pstoreu(px+PacketSize, ei_padd(ei_pmul(pc,xi1),pcj.pmul(ps,yi1)));
ei_pstore (py, ei_psub(ei_pmul(pc,yi),ei_pmul(ps,xi)));
ei_pstore (py+PacketSize, ei_psub(ei_pmul(pc,yi1),ei_pmul(ps,xi1)));
px += Peeling*PacketSize;
@@ -369,7 +369,7 @@ void /*EIGEN_DONT_INLINE*/ ei_apply_rotation_in_the_plane(VectorX& _x, VectorY&
{
Packet xi = ei_ploadu(x+peelingEnd);
Packet yi = ei_pload (y+peelingEnd);
- ei_pstoreu(x+peelingEnd, ei_padd(ei_pmul(pc,xi),cj.pmul(ps,yi)));
+ ei_pstoreu(x+peelingEnd, ei_padd(ei_pmul(pc,xi),pcj.pmul(ps,yi)));
ei_pstore (y+peelingEnd, ei_psub(ei_pmul(pc,yi),ei_pmul(ps,xi)));
}
}
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-07 03:40:28 +0000