From c79bd4b64bcc7c66d2b53a9a668a3310a3ae2998 Mon Sep 17 00:00:00 2001 From: Gael Guennebaud Date: Fri, 25 Apr 2014 11:06:03 +0200 Subject: Minor optimizations in product kernel: - use pbroadcast4 (helpful when AVX is not available) - process all remaining rows at once (significant speedup for small matrices) --- Eigen/src/Core/products/GeneralBlockPanelKernel.h | 162 ++++++++++------------ 1 file changed, 73 insertions(+), 89 deletions(-) (limited to 'Eigen') diff --git a/Eigen/src/Core/products/GeneralBlockPanelKernel.h b/Eigen/src/Core/products/GeneralBlockPanelKernel.h index 0a94f25e4..e76c12c39 100644 --- a/Eigen/src/Core/products/GeneralBlockPanelKernel.h +++ b/Eigen/src/Core/products/GeneralBlockPanelKernel.h @@ -214,11 +214,11 @@ public: p = pset1(ResScalar(0)); } -// EIGEN_STRONG_INLINE void broadcastRhs(const RhsScalar* b, RhsPacket& b0, RhsPacket& b1, RhsPacket& b2, RhsPacket& b3) -// { -// pbroadcast4(b, b0, b1, b2, b3); -// } -// + EIGEN_STRONG_INLINE void broadcastRhs(const RhsScalar* b, RhsPacket& b0, RhsPacket& b1, RhsPacket& b2, RhsPacket& b3) + { + pbroadcast4(b, b0, b1, b2, b3); + } + // EIGEN_STRONG_INLINE void broadcastRhs(const RhsScalar* b, RhsPacket& b0, RhsPacket& b1) // { // pbroadcast2(b, b0, b1); @@ -342,11 +342,11 @@ public: dest = ploadu(a); } -// EIGEN_STRONG_INLINE void broadcastRhs(const RhsScalar* b, RhsPacket& b0, RhsPacket& b1, RhsPacket& b2, RhsPacket& b3) -// { -// pbroadcast4(b, b0, b1, b2, b3); -// } -// + EIGEN_STRONG_INLINE void broadcastRhs(const RhsScalar* b, RhsPacket& b0, RhsPacket& b1, RhsPacket& b2, RhsPacket& b3) + { + pbroadcast4(b, b0, b1, b2, b3); + } + // EIGEN_STRONG_INLINE void broadcastRhs(const RhsScalar* b, RhsPacket& b0, RhsPacket& b1) // { // pbroadcast2(b, b0, b1); @@ -713,11 +713,9 @@ void gebp_kernel const Index peeled_mc1 = mr>=1*Traits::LhsProgress ? (rows/(1*LhsProgress))*(1*LhsProgress) : 0; enum { pk = 8 }; // NOTE Such a large peeling factor is important for large matrices (~ +5% when >1000 on Haswell) const Index peeled_kc = depth & ~(pk-1); + const Index prefetch_res_offset = 32/sizeof(ResScalar); // const Index depth2 = depth & ~1; -// std::cout << mr << " " << peeled_mc3 << " " << peeled_mc2 << " " << peeled_mc1 << "\n"; - - //---------- Process 3 * LhsProgress rows at once ---------- // This corresponds to 3*LhsProgress x nr register blocks. // Usually, make sense only with FMA @@ -736,19 +734,12 @@ void gebp_kernel prefetch(&blA[0]); // gets res block as register - AccPacket C0, C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11; - traits.initAcc(C0); - traits.initAcc(C1); - traits.initAcc(C2); - traits.initAcc(C3); - traits.initAcc(C4); - traits.initAcc(C5); - traits.initAcc(C6); - traits.initAcc(C7); - traits.initAcc(C8); - traits.initAcc(C9); - traits.initAcc(C10); - traits.initAcc(C11); + AccPacket C0, C1, C2, C3, + C4, C5, C6, C7, + C8, C9, C10, C11; + traits.initAcc(C0); traits.initAcc(C1); traits.initAcc(C2); traits.initAcc(C3); + traits.initAcc(C4); traits.initAcc(C5); traits.initAcc(C6); traits.initAcc(C7); + traits.initAcc(C8); traits.initAcc(C9); traits.initAcc(C10); traits.initAcc(C11); ResScalar* r0 = &res[(j2+0)*resStride + i]; ResScalar* r1 = &res[(j2+1)*resStride + i]; @@ -767,7 +758,6 @@ void gebp_kernel for(Index k=0; k prefetch(&blA[0]); // gets res block as register - AccPacket C0, C1, C2, C3, C4, C5, C6, C7; - traits.initAcc(C0); - traits.initAcc(C1); - traits.initAcc(C2); - traits.initAcc(C3); - traits.initAcc(C4); - traits.initAcc(C5); - traits.initAcc(C6); - traits.initAcc(C7); + AccPacket C0, C1, C2, C3, + C4, C5, C6, C7; + traits.initAcc(C0); traits.initAcc(C1); traits.initAcc(C2); traits.initAcc(C3); + traits.initAcc(C4); traits.initAcc(C5); traits.initAcc(C6); traits.initAcc(C7); ResScalar* r0 = &res[(j2+0)*resStride + i]; ResScalar* r1 = &res[(j2+1)*resStride + i]; ResScalar* r2 = &res[(j2+2)*resStride + i]; ResScalar* r3 = &res[(j2+3)*resStride + i]; - internal::prefetch(r0); - internal::prefetch(r1); - internal::prefetch(r2); - internal::prefetch(r3); + internal::prefetch(r0+prefetch_res_offset); + internal::prefetch(r1+prefetch_res_offset); + internal::prefetch(r2+prefetch_res_offset); + internal::prefetch(r3+prefetch_res_offset); // performs "inner" products const RhsScalar* blB = &blockB[j2*strideB+offsetB*nr]; @@ -977,26 +962,22 @@ void gebp_kernel for(Index k=0; k blB += pk*4*RhsProgress; blA += pk*(2*Traits::LhsProgress); - IACA_END } // process remaining peeled loop for(Index k=peeled_kc; k traits.initAcc(C4); ResScalar* r0 = &res[(j2+0)*resStride + i]; + internal::prefetch(r0+prefetch_res_offset); // performs "inner" products const RhsScalar* blB = &blockB[j2*strideB+offsetB]; @@ -1079,7 +1060,7 @@ void gebp_kernel traits.loadLhs(&blA[(0+2*K)*LhsProgress], A0); \ traits.loadLhs(&blA[(1+2*K)*LhsProgress], A1); \ traits.loadRhs(&blB[(0+K)*RhsProgress], B_0); \ - traits.madd(A0, B_0, C0, B1); \ + traits.madd(A0, B_0, C0, B1); \ traits.madd(A1, B_0, C4, B_0) EIGEN_GEBGP_ONESTEP(0); @@ -1143,10 +1124,10 @@ void gebp_kernel ResScalar* r2 = &res[(j2+2)*resStride + i]; ResScalar* r3 = &res[(j2+3)*resStride + i]; - internal::prefetch(r0); - internal::prefetch(r1); - internal::prefetch(r2); - internal::prefetch(r3); + internal::prefetch(r0+prefetch_res_offset); + internal::prefetch(r1+prefetch_res_offset); + internal::prefetch(r2+prefetch_res_offset); + internal::prefetch(r3+prefetch_res_offset); // performs "inner" products const RhsScalar* blB = &blockB[j2*strideB+offsetB*nr]; @@ -1155,19 +1136,18 @@ void gebp_kernel for(Index k=0; k blB += pk*4*RhsProgress; blA += pk*1*LhsProgress; - IACA_END } // process remaining peeled loop for(Index k=peeled_kc; k } //---------- Process remaining rows, 1 at once ---------- { - // loop on each row of the lhs (1*LhsProgress x depth) - for(Index i=peeled_mc1; i Index k=0; for(; k res[(j2+3)*resStride + i] += alpha*C3; } } - // remaining columns - for(Index j2=packet_cols4; j2