From fd1dcb6b45a2c797ad4c4d6cc7678ee70763b4ed Mon Sep 17 00:00:00 2001 From: Antonio Sanchez Date: Fri, 20 Nov 2020 08:49:57 -0800 Subject: Fixes duplicate symbol when building blas Missing inline breaks blas, since symbol generated in `complex_single.cpp`, `complex_double.cpp`, `single.cpp`, `double.cpp` Changed rest of inlines to `EIGEN_STRONG_INLINE`. --- Eigen/src/Core/arch/NEON/PacketMath.h | 216 +++++++++++++++++----------------- 1 file changed, 108 insertions(+), 108 deletions(-) (limited to 'Eigen/src/Core/arch/NEON/PacketMath.h') diff --git a/Eigen/src/Core/arch/NEON/PacketMath.h b/Eigen/src/Core/arch/NEON/PacketMath.h index b13cbe5ec..709cebe4e 100644 --- a/Eigen/src/Core/arch/NEON/PacketMath.h +++ b/Eigen/src/Core/arch/NEON/PacketMath.h @@ -87,8 +87,8 @@ typedef uint64x2_t Packet2ul; // fuctionally equivalent to _mm_shuffle_ps in SSE when interleave // == false (i.e. shuffle(m, n, mask) equals _mm_shuffle_ps(m, n, mask)), // interleave m and n when interleave == true. Currently used in LU/arch/InverseSize4.h -// to enable a shared implementation for fast inversion of matrices of size 4. -template +// to enable a shared implementation for fast inversion of matrices of size 4. +template EIGEN_STRONG_INLINE Packet4f shuffle(const Packet4f &m, const Packet4f &n, int mask) { const float* a = reinterpret_cast(&m); @@ -97,8 +97,8 @@ EIGEN_STRONG_INLINE Packet4f shuffle(const Packet4f &m, const Packet4f &n, int m return res; } -template<> -EIGEN_STRONG_INLINE Packet4f shuffle(const Packet4f &m, const Packet4f &n, int mask) +template<> +EIGEN_STRONG_INLINE Packet4f shuffle(const Packet4f &m, const Packet4f &n, int mask) { const float* a = reinterpret_cast(&m); const float* b = reinterpret_cast(&n); @@ -109,7 +109,7 @@ EIGEN_STRONG_INLINE Packet4f shuffle(const Packet4f &m, const Packet4f &n, EIGEN_STRONG_INLINE static int eigen_neon_shuffle_mask(int p, int q, int r, int s) {return ((s)<<6|(r)<<4|(q)<<2|(p));} EIGEN_STRONG_INLINE Packet4f vec4f_swizzle2(const Packet4f& a, const Packet4f& b, int p, int q, int r, int s) -{ +{ return shuffle(a,b,eigen_neon_shuffle_mask(p, q, r, s)); } EIGEN_STRONG_INLINE Packet4f vec4f_movelh(const Packet4f& a, const Packet4f& b) @@ -1922,13 +1922,13 @@ template<> EIGEN_STRONG_INLINE void pstoreu(int64_t* to, const Packet2l template<> EIGEN_STRONG_INLINE void pstoreu(uint64_t* to, const Packet2ul& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_u64(to,from); } -template<> EIGEN_DEVICE_FUNC inline Packet2f pgather(const float* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f pgather(const float* from, Index stride) { Packet2f res = vld1_dup_f32(from); res = vld1_lane_f32(from + 1*stride, res, 1); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet4f pgather(const float* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pgather(const float* from, Index stride) { Packet4f res = vld1q_dup_f32(from); res = vld1q_lane_f32(from + 1*stride, res, 1); @@ -1936,14 +1936,14 @@ template<> EIGEN_DEVICE_FUNC inline Packet4f pgather(const floa res = vld1q_lane_f32(from + 3*stride, res, 3); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet4c pgather(const int8_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c pgather(const int8_t* from, Index stride) { Packet4c res; for (int i = 0; i != 4; i++) reinterpret_cast(&res)[i] = *(from + i * stride); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet8c pgather(const int8_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c pgather(const int8_t* from, Index stride) { Packet8c res = vld1_dup_s8(from); res = vld1_lane_s8(from + 1*stride, res, 1); @@ -1955,7 +1955,7 @@ template<> EIGEN_DEVICE_FUNC inline Packet8c pgather(const int res = vld1_lane_s8(from + 7*stride, res, 7); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet16c pgather(const int8_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pgather(const int8_t* from, Index stride) { Packet16c res = vld1q_dup_s8(from); res = vld1q_lane_s8(from + 1*stride, res, 1); @@ -1975,14 +1975,14 @@ template<> EIGEN_DEVICE_FUNC inline Packet16c pgather(const i res = vld1q_lane_s8(from + 15*stride, res, 15); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet4uc pgather(const uint8_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc pgather(const uint8_t* from, Index stride) { Packet4uc res; for (int i = 0; i != 4; i++) reinterpret_cast(&res)[i] = *(from + i * stride); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet8uc pgather(const uint8_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc pgather(const uint8_t* from, Index stride) { Packet8uc res = vld1_dup_u8(from); res = vld1_lane_u8(from + 1*stride, res, 1); @@ -1994,7 +1994,7 @@ template<> EIGEN_DEVICE_FUNC inline Packet8uc pgather(const res = vld1_lane_u8(from + 7*stride, res, 7); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet16uc pgather(const uint8_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pgather(const uint8_t* from, Index stride) { Packet16uc res = vld1q_dup_u8(from); res = vld1q_lane_u8(from + 1*stride, res, 1); @@ -2014,7 +2014,7 @@ template<> EIGEN_DEVICE_FUNC inline Packet16uc pgather(cons res = vld1q_lane_u8(from + 15*stride, res, 15); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet4s pgather(const int16_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s pgather(const int16_t* from, Index stride) { Packet4s res = vld1_dup_s16(from); res = vld1_lane_s16(from + 1*stride, res, 1); @@ -2022,7 +2022,7 @@ template<> EIGEN_DEVICE_FUNC inline Packet4s pgather(const in res = vld1_lane_s16(from + 3*stride, res, 3); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet8s pgather(const int16_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pgather(const int16_t* from, Index stride) { Packet8s res = vld1q_dup_s16(from); res = vld1q_lane_s16(from + 1*stride, res, 1); @@ -2034,7 +2034,7 @@ template<> EIGEN_DEVICE_FUNC inline Packet8s pgather(const in res = vld1q_lane_s16(from + 7*stride, res, 7); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet4us pgather(const uint16_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us pgather(const uint16_t* from, Index stride) { Packet4us res = vld1_dup_u16(from); res = vld1_lane_u16(from + 1*stride, res, 1); @@ -2042,7 +2042,7 @@ template<> EIGEN_DEVICE_FUNC inline Packet4us pgather(const res = vld1_lane_u16(from + 3*stride, res, 3); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet8us pgather(const uint16_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pgather(const uint16_t* from, Index stride) { Packet8us res = vld1q_dup_u16(from); res = vld1q_lane_u16(from + 1*stride, res, 1); @@ -2054,13 +2054,13 @@ template<> EIGEN_DEVICE_FUNC inline Packet8us pgather(const res = vld1q_lane_u16(from + 7*stride, res, 7); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet2i pgather(const int32_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i pgather(const int32_t* from, Index stride) { Packet2i res = vld1_dup_s32(from); res = vld1_lane_s32(from + 1*stride, res, 1); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet4i pgather(const int32_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pgather(const int32_t* from, Index stride) { Packet4i res = vld1q_dup_s32(from); res = vld1q_lane_s32(from + 1*stride, res, 1); @@ -2068,13 +2068,13 @@ template<> EIGEN_DEVICE_FUNC inline Packet4i pgather(const in res = vld1q_lane_s32(from + 3*stride, res, 3); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet2ui pgather(const uint32_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui pgather(const uint32_t* from, Index stride) { Packet2ui res = vld1_dup_u32(from); res = vld1_lane_u32(from + 1*stride, res, 1); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet4ui pgather(const uint32_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pgather(const uint32_t* from, Index stride) { Packet4ui res = vld1q_dup_u32(from); res = vld1q_lane_u32(from + 1*stride, res, 1); @@ -2082,37 +2082,37 @@ template<> EIGEN_DEVICE_FUNC inline Packet4ui pgather(const res = vld1q_lane_u32(from + 3*stride, res, 3); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet2l pgather(const int64_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pgather(const int64_t* from, Index stride) { Packet2l res = vld1q_dup_s64(from); res = vld1q_lane_s64(from + 1*stride, res, 1); return res; } -template<> EIGEN_DEVICE_FUNC inline Packet2ul pgather(const uint64_t* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pgather(const uint64_t* from, Index stride) { Packet2ul res = vld1q_dup_u64(from); res = vld1q_lane_u64(from + 1*stride, res, 1); return res; } -template<> EIGEN_DEVICE_FUNC inline void pscatter(float* to, const Packet2f& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(float* to, const Packet2f& from, Index stride) { vst1_lane_f32(to + stride*0, from, 0); vst1_lane_f32(to + stride*1, from, 1); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(float* to, const Packet4f& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(float* to, const Packet4f& from, Index stride) { vst1q_lane_f32(to + stride*0, from, 0); vst1q_lane_f32(to + stride*1, from, 1); vst1q_lane_f32(to + stride*2, from, 2); vst1q_lane_f32(to + stride*3, from, 3); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(int8_t* to, const Packet4c& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int8_t* to, const Packet4c& from, Index stride) { for (int i = 0; i != 4; i++) *(to + i * stride) = reinterpret_cast(&from)[i]; } -template<> EIGEN_DEVICE_FUNC inline void pscatter(int8_t* to, const Packet8c& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int8_t* to, const Packet8c& from, Index stride) { vst1_lane_s8(to + stride*0, from, 0); vst1_lane_s8(to + stride*1, from, 1); @@ -2123,7 +2123,7 @@ template<> EIGEN_DEVICE_FUNC inline void pscatter(int8_t* to, vst1_lane_s8(to + stride*6, from, 6); vst1_lane_s8(to + stride*7, from, 7); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(int8_t* to, const Packet16c& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int8_t* to, const Packet16c& from, Index stride) { vst1q_lane_s8(to + stride*0, from, 0); vst1q_lane_s8(to + stride*1, from, 1); @@ -2142,12 +2142,12 @@ template<> EIGEN_DEVICE_FUNC inline void pscatter(int8_t* to, vst1q_lane_s8(to + stride*14, from, 14); vst1q_lane_s8(to + stride*15, from, 15); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(uint8_t* to, const Packet4uc& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint8_t* to, const Packet4uc& from, Index stride) { for (int i = 0; i != 4; i++) *(to + i * stride) = reinterpret_cast(&from)[i]; } -template<> EIGEN_DEVICE_FUNC inline void pscatter(uint8_t* to, const Packet8uc& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint8_t* to, const Packet8uc& from, Index stride) { vst1_lane_u8(to + stride*0, from, 0); vst1_lane_u8(to + stride*1, from, 1); @@ -2158,7 +2158,7 @@ template<> EIGEN_DEVICE_FUNC inline void pscatter(uint8_t* t vst1_lane_u8(to + stride*6, from, 6); vst1_lane_u8(to + stride*7, from, 7); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(uint8_t* to, const Packet16uc& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint8_t* to, const Packet16uc& from, Index stride) { vst1q_lane_u8(to + stride*0, from, 0); vst1q_lane_u8(to + stride*1, from, 1); @@ -2177,14 +2177,14 @@ template<> EIGEN_DEVICE_FUNC inline void pscatter(uint8_t* vst1q_lane_u8(to + stride*14, from, 14); vst1q_lane_u8(to + stride*15, from, 15); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(int16_t* to, const Packet4s& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int16_t* to, const Packet4s& from, Index stride) { vst1_lane_s16(to + stride*0, from, 0); vst1_lane_s16(to + stride*1, from, 1); vst1_lane_s16(to + stride*2, from, 2); vst1_lane_s16(to + stride*3, from, 3); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(int16_t* to, const Packet8s& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int16_t* to, const Packet8s& from, Index stride) { vst1q_lane_s16(to + stride*0, from, 0); vst1q_lane_s16(to + stride*1, from, 1); @@ -2195,14 +2195,14 @@ template<> EIGEN_DEVICE_FUNC inline void pscatter(int16_t* to vst1q_lane_s16(to + stride*6, from, 6); vst1q_lane_s16(to + stride*7, from, 7); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(uint16_t* to, const Packet4us& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint16_t* to, const Packet4us& from, Index stride) { vst1_lane_u16(to + stride*0, from, 0); vst1_lane_u16(to + stride*1, from, 1); vst1_lane_u16(to + stride*2, from, 2); vst1_lane_u16(to + stride*3, from, 3); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(uint16_t* to, const Packet8us& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint16_t* to, const Packet8us& from, Index stride) { vst1q_lane_u16(to + stride*0, from, 0); vst1q_lane_u16(to + stride*1, from, 1); @@ -2213,36 +2213,36 @@ template<> EIGEN_DEVICE_FUNC inline void pscatter(uint16_t* vst1q_lane_u16(to + stride*6, from, 6); vst1q_lane_u16(to + stride*7, from, 7); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(int32_t* to, const Packet2i& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int32_t* to, const Packet2i& from, Index stride) { vst1_lane_s32(to + stride*0, from, 0); vst1_lane_s32(to + stride*1, from, 1); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(int32_t* to, const Packet4i& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int32_t* to, const Packet4i& from, Index stride) { vst1q_lane_s32(to + stride*0, from, 0); vst1q_lane_s32(to + stride*1, from, 1); vst1q_lane_s32(to + stride*2, from, 2); vst1q_lane_s32(to + stride*3, from, 3); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(uint32_t* to, const Packet2ui& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint32_t* to, const Packet2ui& from, Index stride) { vst1_lane_u32(to + stride*0, from, 0); vst1_lane_u32(to + stride*1, from, 1); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(uint32_t* to, const Packet4ui& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint32_t* to, const Packet4ui& from, Index stride) { vst1q_lane_u32(to + stride*0, from, 0); vst1q_lane_u32(to + stride*1, from, 1); vst1q_lane_u32(to + stride*2, from, 2); vst1q_lane_u32(to + stride*3, from, 3); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(int64_t* to, const Packet2l& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int64_t* to, const Packet2l& from, Index stride) { vst1q_lane_s64(to + stride*0, from, 0); vst1q_lane_s64(to + stride*1, from, 1); } -template<> EIGEN_DEVICE_FUNC inline void pscatter(uint64_t* to, const Packet2ul& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint64_t* to, const Packet2ul& from, Index stride) { vst1q_lane_u64(to + stride*0, from, 0); vst1q_lane_u64(to + stride*1, from, 1); @@ -2457,23 +2457,23 @@ template<> EIGEN_STRONG_INLINE int64_t predux(const Packet2l& a) template<> EIGEN_STRONG_INLINE uint64_t predux(const Packet2ul& a) { return vgetq_lane_u64(a, 0) + vgetq_lane_u64(a, 1); } -template<> EIGEN_DEVICE_FUNC inline Packet4c predux_half_dowto4(const Packet8c& a) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c predux_half_dowto4(const Packet8c& a) { return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(a, vreinterpret_s8_s32(vrev64_s32(vreinterpret_s32_s8(a))))), 0); } -template<> EIGEN_DEVICE_FUNC inline Packet8c predux_half_dowto4(const Packet16c& a) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c predux_half_dowto4(const Packet16c& a) { return vadd_s8(vget_high_s8(a), vget_low_s8(a)); } -template<> EIGEN_DEVICE_FUNC inline Packet4uc predux_half_dowto4(const Packet8uc& a) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc predux_half_dowto4(const Packet8uc& a) { return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(a, vreinterpret_u8_u32(vrev64_u32(vreinterpret_u32_u8(a))))), 0); } -template<> EIGEN_DEVICE_FUNC inline Packet8uc predux_half_dowto4(const Packet16uc& a) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc predux_half_dowto4(const Packet16uc& a) { return vadd_u8(vget_high_u8(a), vget_low_u8(a)); } -template<> EIGEN_DEVICE_FUNC inline Packet4s predux_half_dowto4(const Packet8s& a) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s predux_half_dowto4(const Packet8s& a) { return vadd_s16(vget_high_s16(a), vget_low_s16(a)); } -template<> EIGEN_DEVICE_FUNC inline Packet4us predux_half_dowto4(const Packet8us& a) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us predux_half_dowto4(const Packet8us& a) { return vadd_u16(vget_high_u16(a), vget_low_u16(a)); } // Other reduction functions: @@ -2752,13 +2752,13 @@ template<> EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x) return vget_lane_u32(vpmax_u32(tmp, tmp), 0); } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const float32x2x2_t z = vzip_f32(kernel.packet[0], kernel.packet[1]); kernel.packet[0] = z.val[0]; kernel.packet[1] = z.val[1]; } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const float32x4x2_t tmp1 = vzipq_f32(kernel.packet[0], kernel.packet[1]); const float32x4x2_t tmp2 = vzipq_f32(kernel.packet[2], kernel.packet[3]); @@ -2768,7 +2768,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) kernel.packet[2] = vcombine_f32(vget_low_f32(tmp1.val[1]), vget_low_f32(tmp2.val[1])); kernel.packet[3] = vcombine_f32(vget_high_f32(tmp1.val[1]), vget_high_f32(tmp2.val[1])); } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const int8x8_t a = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[2], vdup_n_s32(kernel.packet[0]), 1)); const int8x8_t b = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[3], vdup_n_s32(kernel.packet[1]), 1)); @@ -2781,7 +2781,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) kernel.packet[2] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 0); kernel.packet[3] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 1); } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { int8x8x2_t zip8[4]; uint16x4x2_t zip16[4]; @@ -2811,7 +2811,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) } } } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { int8x16x2_t zip8[8]; uint16x8x2_t zip16[8]; @@ -2858,7 +2858,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) } } } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const uint8x8_t a = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[2], vdup_n_u32(kernel.packet[0]), 1)); const uint8x8_t b = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[3], vdup_n_u32(kernel.packet[1]), 1)); @@ -2871,7 +2871,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) kernel.packet[2] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 0); kernel.packet[3] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 1); } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { uint8x8x2_t zip8[4]; uint16x4x2_t zip16[4]; @@ -2901,7 +2901,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) } } } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { uint8x16x2_t zip8[8]; uint16x8x2_t zip16[8]; @@ -2946,7 +2946,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) } } } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const int16x4x2_t zip16_1 = vzip_s16(kernel.packet[0], kernel.packet[1]); const int16x4x2_t zip16_2 = vzip_s16(kernel.packet[2], kernel.packet[3]); @@ -2960,7 +2960,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) kernel.packet[3] = vreinterpret_s16_u32(zip32_2.val[1]); } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const int16x8x2_t zip16_1 = vzipq_s16(kernel.packet[0], kernel.packet[1]); const int16x8x2_t zip16_2 = vzipq_s16(kernel.packet[2], kernel.packet[3]); @@ -2974,7 +2974,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) kernel.packet[3] = vreinterpretq_s16_u32(zip32_2.val[1]); } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const uint8x16x2_t zip8_1 = vzipq_u8(kernel.packet[0], kernel.packet[1]); const uint8x16x2_t zip8_2 = vzipq_u8(kernel.packet[2], kernel.packet[3]); @@ -2988,7 +2988,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) kernel.packet[3] = vreinterpretq_u8_u16(zip16_2.val[1]); } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const int16x8x2_t zip16_1 = vzipq_s16(kernel.packet[0], kernel.packet[1]); const int16x8x2_t zip16_2 = vzipq_s16(kernel.packet[2], kernel.packet[3]); @@ -3009,7 +3009,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) kernel.packet[6] = vreinterpretq_s16_u32(vcombine_u32(vget_low_u32(zip32_2.val[1]), vget_low_u32(zip32_4.val[1]))); kernel.packet[7] = vreinterpretq_s16_u32(vcombine_u32(vget_high_u32(zip32_2.val[1]), vget_high_u32(zip32_4.val[1]))); } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const uint16x4x2_t zip16_1 = vzip_u16(kernel.packet[0], kernel.packet[1]); const uint16x4x2_t zip16_2 = vzip_u16(kernel.packet[2], kernel.packet[3]); @@ -3022,7 +3022,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) kernel.packet[2] = vreinterpret_u16_u32(zip32_2.val[0]); kernel.packet[3] = vreinterpret_u16_u32(zip32_2.val[1]); } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const uint16x8x2_t zip16_1 = vzipq_u16(kernel.packet[0], kernel.packet[1]); const uint16x8x2_t zip16_2 = vzipq_u16(kernel.packet[2], kernel.packet[3]); @@ -3043,13 +3043,13 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) kernel.packet[6] = vreinterpretq_u16_u32(vcombine_u32(vget_low_u32(zip32_2.val[1]), vget_low_u32(zip32_4.val[1]))); kernel.packet[7] = vreinterpretq_u16_u32(vcombine_u32(vget_high_u32(zip32_2.val[1]), vget_high_u32(zip32_4.val[1]))); } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const int32x2x2_t z = vzip_s32(kernel.packet[0], kernel.packet[1]); kernel.packet[0] = z.val[0]; kernel.packet[1] = z.val[1]; } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const int32x4x2_t tmp1 = vzipq_s32(kernel.packet[0], kernel.packet[1]); const int32x4x2_t tmp2 = vzipq_s32(kernel.packet[2], kernel.packet[3]); @@ -3059,13 +3059,13 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) kernel.packet[2] = vcombine_s32(vget_low_s32(tmp1.val[1]), vget_low_s32(tmp2.val[1])); kernel.packet[3] = vcombine_s32(vget_high_s32(tmp1.val[1]), vget_high_s32(tmp2.val[1])); } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const uint32x2x2_t z = vzip_u32(kernel.packet[0], kernel.packet[1]); kernel.packet[0] = z.val[0]; kernel.packet[1] = z.val[1]; } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const uint32x4x2_t tmp1 = vzipq_u32(kernel.packet[0], kernel.packet[1]); const uint32x4x2_t tmp2 = vzipq_u32(kernel.packet[2], kernel.packet[3]); @@ -3075,7 +3075,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) kernel.packet[2] = vcombine_u32(vget_low_u32(tmp1.val[1]), vget_low_u32(tmp2.val[1])); kernel.packet[3] = vcombine_u32(vget_high_u32(tmp1.val[1]), vget_high_u32(tmp2.val[1])); } -EIGEN_DEVICE_FUNC inline void +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { #if EIGEN_ARCH_ARM64 @@ -3094,7 +3094,7 @@ ptranspose(PacketBlock& kernel) kernel.packet[1] = vcombine_s64(tmp[0][1], tmp[1][1]); #endif } -EIGEN_DEVICE_FUNC inline void +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { #if EIGEN_ARCH_ARM64 @@ -3114,37 +3114,37 @@ ptranspose(PacketBlock& kernel) #endif } -template<> EIGEN_DEVICE_FUNC inline Packet2f pselect( const Packet2f& mask, const Packet2f& a, const Packet2f& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f pselect( const Packet2f& mask, const Packet2f& a, const Packet2f& b) { return vbsl_f32(vreinterpret_u32_f32(mask), a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet4f pselect(const Packet4f& mask, const Packet4f& a, const Packet4f& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pselect(const Packet4f& mask, const Packet4f& a, const Packet4f& b) { return vbslq_f32(vreinterpretq_u32_f32(mask), a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet8c pselect(const Packet8c& mask, const Packet8c& a, const Packet8c& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c pselect(const Packet8c& mask, const Packet8c& a, const Packet8c& b) { return vbsl_s8(vreinterpret_u8_s8(mask), a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet16c pselect(const Packet16c& mask, const Packet16c& a, const Packet16c& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pselect(const Packet16c& mask, const Packet16c& a, const Packet16c& b) { return vbslq_s8(vreinterpretq_u8_s8(mask), a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet8uc pselect(const Packet8uc& mask, const Packet8uc& a, const Packet8uc& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc pselect(const Packet8uc& mask, const Packet8uc& a, const Packet8uc& b) { return vbsl_u8(mask, a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet16uc pselect(const Packet16uc& mask, const Packet16uc& a, const Packet16uc& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pselect(const Packet16uc& mask, const Packet16uc& a, const Packet16uc& b) { return vbslq_u8(mask, a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet4s pselect(const Packet4s& mask, const Packet4s& a, const Packet4s& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s pselect(const Packet4s& mask, const Packet4s& a, const Packet4s& b) { return vbsl_s16(vreinterpret_u16_s16(mask), a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet8s pselect(const Packet8s& mask, const Packet8s& a, const Packet8s& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pselect(const Packet8s& mask, const Packet8s& a, const Packet8s& b) { return vbslq_s16(vreinterpretq_u16_s16(mask), a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet4us pselect(const Packet4us& mask, const Packet4us& a, const Packet4us& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us pselect(const Packet4us& mask, const Packet4us& a, const Packet4us& b) { return vbsl_u16(mask, a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet8us pselect(const Packet8us& mask, const Packet8us& a, const Packet8us& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pselect(const Packet8us& mask, const Packet8us& a, const Packet8us& b) { return vbslq_u16(mask, a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet2i pselect(const Packet2i& mask, const Packet2i& a, const Packet2i& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i pselect(const Packet2i& mask, const Packet2i& a, const Packet2i& b) { return vbsl_s32(vreinterpret_u32_s32(mask), a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet4i pselect(const Packet4i& mask, const Packet4i& a, const Packet4i& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pselect(const Packet4i& mask, const Packet4i& a, const Packet4i& b) { return vbslq_s32(vreinterpretq_u32_s32(mask), a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet2ui pselect(const Packet2ui& mask, const Packet2ui& a, const Packet2ui& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui pselect(const Packet2ui& mask, const Packet2ui& a, const Packet2ui& b) { return vbsl_u32(mask, a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet4ui pselect(const Packet4ui& mask, const Packet4ui& a, const Packet4ui& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pselect(const Packet4ui& mask, const Packet4ui& a, const Packet4ui& b) { return vbslq_u32(mask, a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet2l pselect(const Packet2l& mask, const Packet2l& a, const Packet2l& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pselect(const Packet2l& mask, const Packet2l& a, const Packet2l& b) { return vbslq_s64(vreinterpretq_u64_s64(mask), a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet2ul pselect(const Packet2ul& mask, const Packet2ul& a, const Packet2ul& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pselect(const Packet2ul& mask, const Packet2ul& a, const Packet2ul& b) { return vbslq_u64(mask, a, b); } /** @@ -3254,7 +3254,7 @@ template<> EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& _x){ vcltq_f32(_x, pset1((std::numeric_limits::min)()))); // Compute approximate reciprocal sqrt. Packet4f x = vrsqrteq_f32(_x); - // Do a single step of Newton's iteration. + // Do a single step of Newton's iteration. //the number 1.5f was set reference to Quake3's fast inverse square root x = vmulq_f32(x, psub(pset1(1.5f), pmul(half, pmul(x, x)))); // Flush results for denormals to zero. @@ -3273,7 +3273,7 @@ template<> EIGEN_STRONG_INLINE Packet2f psqrt(const Packet2f& _x){ return vreinterpret_f32_u32(vbic_u32(vreinterpret_u32_f32(pmul(_x, x)), denormal_mask)); } -#else +#else template<> EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& _x){return vsqrtq_f32(_x);} template<> EIGEN_STRONG_INLINE Packet2f psqrt(const Packet2f& _x){return vsqrt_f32(_x); } #endif @@ -3441,7 +3441,7 @@ template<> EIGEN_STRONG_INLINE Packet4bf pandnot(const Packet4bf& a,const Packet return pandnot(a, b); } -template<> EIGEN_DEVICE_FUNC inline Packet4bf pselect(const Packet4bf& mask, const Packet4bf& a, +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4bf pselect(const Packet4bf& mask, const Packet4bf& a, const Packet4bf& b) { return pselect(mask, a, b); @@ -3507,7 +3507,7 @@ template<> EIGEN_STRONG_INLINE Packet4bf preverse(const Packet4bf& a) return preverse(a); } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { PacketBlock k; k.packet[0] = kernel.packet[0]; @@ -3573,7 +3573,7 @@ typedef float64x2_t Packet2d; typedef float64x1_t Packet1d; // fuctionally equivalent to _mm_shuffle_pd in SSE (i.e. shuffle(m, n, mask) equals _mm_shuffle_pd(m,n,mask)) -// Currently used in LU/arch/InverseSize4.h to enable a shared implementation +// Currently used in LU/arch/InverseSize4.h to enable a shared implementation // for fast inversion of matrices of size 4. EIGEN_STRONG_INLINE Packet2d shuffle(const Packet2d& m, const Packet2d& n, int mask) { @@ -3739,7 +3739,7 @@ template<> EIGEN_STRONG_INLINE void pstore(double* to, const Packet2d& f template<> EIGEN_STRONG_INLINE void pstoreu(double* to, const Packet2d& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f64(to,from); } -template<> EIGEN_DEVICE_FUNC inline Packet2d pgather(const double* from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pgather(const double* from, Index stride) { Packet2d res = pset1(0.0); res = vld1q_lane_f64(from + 0*stride, res, 0); @@ -3747,7 +3747,7 @@ template<> EIGEN_DEVICE_FUNC inline Packet2d pgather(const dou return res; } -template<> EIGEN_DEVICE_FUNC inline void pscatter(double* to, const Packet2d& from, Index stride) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(double* to, const Packet2d& from, Index stride) { vst1q_lane_f64(to + stride*0, from, 0); vst1q_lane_f64(to + stride*1, from, 1); @@ -3791,7 +3791,7 @@ template<> EIGEN_STRONG_INLINE double predux_max(const Packet2d& a) { return vgetq_lane_f64(vpmaxq_f64(a,a), 0); } -EIGEN_DEVICE_FUNC inline void +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { const float64x2_t tmp1 = vzip1q_f64(kernel.packet[0], kernel.packet[1]); @@ -3801,7 +3801,7 @@ ptranspose(PacketBlock& kernel) kernel.packet[1] = tmp2; } -template<> EIGEN_DEVICE_FUNC inline Packet2d pselect( const Packet2d& mask, const Packet2d& a, const Packet2d& b) +template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pselect( const Packet2d& mask, const Packet2d& a, const Packet2d& b) { return vbslq_f64(vreinterpretq_u64_f64(mask), a, b); } template<> EIGEN_STRONG_INLINE Packet2d pldexp(const Packet2d& a, const Packet2d& exponent) @@ -3829,7 +3829,7 @@ template<> EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& _x){ vcltq_f64(_x, pset1((std::numeric_limits::min)()))); // Compute approximate reciprocal sqrt. Packet2d x = vrsqrteq_f64(_x); - // Do a single step of Newton's iteration. + // Do a single step of Newton's iteration. //the number 1.5f was set reference to Quake3's fast inverse square root x = vmulq_f64(x, psub(pset1(1.5), pmul(half, pmul(x, x)))); // Do one more Newton's iteration to get more accurate result. @@ -3838,7 +3838,7 @@ template<> EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& _x){ return vreinterpretq_f64_u64(vbicq_u64(vreinterpretq_u64_f64(pmul(_x, x)), denormal_mask)); } -#else +#else template<> EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& _x){ return vsqrtq_f64(_x); } #endif @@ -3914,7 +3914,7 @@ struct unpacket_traits { }; template<> -EIGEN_DEVICE_FUNC Packet4hf predux_half_dowto4(const Packet8hf& a) { +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf predux_half_dowto4(const Packet8hf& a) { return vadd_f16(vget_low_f16(a), vget_high_f16(a)); } @@ -4193,23 +4193,23 @@ EIGEN_STRONG_INLINE Packet8hf ploadquad(const Eigen::half* from) { return vcombine_f16(lo, hi); } -EIGEN_DEVICE_FUNC inline Packet8hf pinsertfirst(const Packet8hf& a, Eigen::half b) { return vsetq_lane_f16(b.x, a, 0); } +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pinsertfirst(const Packet8hf& a, Eigen::half b) { return vsetq_lane_f16(b.x, a, 0); } -EIGEN_DEVICE_FUNC inline Packet4hf pinsertfirst(const Packet4hf& a, Eigen::half b) { return vset_lane_f16(b.x, a, 0); } +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pinsertfirst(const Packet4hf& a, Eigen::half b) { return vset_lane_f16(b.x, a, 0); } template <> -EIGEN_DEVICE_FUNC inline Packet8hf pselect(const Packet8hf& mask, const Packet8hf& a, const Packet8hf& b) { +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pselect(const Packet8hf& mask, const Packet8hf& a, const Packet8hf& b) { return vbslq_f16(vreinterpretq_u16_f16(mask), a, b); } template <> -EIGEN_DEVICE_FUNC inline Packet4hf pselect(const Packet4hf& mask, const Packet4hf& a, const Packet4hf& b) { +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pselect(const Packet4hf& mask, const Packet4hf& a, const Packet4hf& b) { return vbsl_f16(vreinterpret_u16_f16(mask), a, b); } -EIGEN_DEVICE_FUNC inline Packet8hf pinsertlast(const Packet8hf& a, Eigen::half b) { return vsetq_lane_f16(b.x, a, 7); } +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pinsertlast(const Packet8hf& a, Eigen::half b) { return vsetq_lane_f16(b.x, a, 7); } -EIGEN_DEVICE_FUNC inline Packet4hf pinsertlast(const Packet4hf& a, Eigen::half b) { return vset_lane_f16(b.x, a, 3); } +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pinsertlast(const Packet4hf& a, Eigen::half b) { return vset_lane_f16(b.x, a, 3); } template <> EIGEN_STRONG_INLINE void pstore(Eigen::half* to, const Packet8hf& from) { @@ -4232,7 +4232,7 @@ EIGEN_STRONG_INLINE void pstoreu(Eigen::half* to, const Packet4hf& } template <> -EIGEN_DEVICE_FUNC inline Packet8hf pgather(const Eigen::half* from, Index stride) { +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pgather(const Eigen::half* from, Index stride) { Packet8hf res = pset1(Eigen::half(0.f)); res = vsetq_lane_f16(from[0 * stride].x, res, 0); res = vsetq_lane_f16(from[1 * stride].x, res, 1); @@ -4246,7 +4246,7 @@ EIGEN_DEVICE_FUNC inline Packet8hf pgather(const Eigen:: } template <> -EIGEN_DEVICE_FUNC inline Packet4hf pgather(const Eigen::half* from, Index stride) { +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pgather(const Eigen::half* from, Index stride) { Packet4hf res = pset1(Eigen::half(0.f)); res = vset_lane_f16(from[0 * stride].x, res, 0); res = vset_lane_f16(from[1 * stride].x, res, 1); @@ -4256,7 +4256,7 @@ EIGEN_DEVICE_FUNC inline Packet4hf pgather(const Eigen:: } template <> -EIGEN_DEVICE_FUNC inline void pscatter(Eigen::half* to, const Packet8hf& from, Index stride) { +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(Eigen::half* to, const Packet8hf& from, Index stride) { to[stride * 0].x = vgetq_lane_f16(from, 0); to[stride * 1].x = vgetq_lane_f16(from, 1); to[stride * 2].x = vgetq_lane_f16(from, 2); @@ -4268,7 +4268,7 @@ EIGEN_DEVICE_FUNC inline void pscatter(Eigen::half* to, } template <> -EIGEN_DEVICE_FUNC inline void pscatter(Eigen::half* to, const Packet4hf& from, Index stride) { +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(Eigen::half* to, const Packet4hf& from, Index stride) { to[stride * 0].x = vget_lane_f16(from, 0); to[stride * 1].x = vget_lane_f16(from, 1); to[stride * 2].x = vget_lane_f16(from, 2); @@ -4422,7 +4422,7 @@ EIGEN_STRONG_INLINE Eigen::half predux_max(const Packet4hf& a) { return h; } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { EIGEN_ALIGN16 Eigen::half in[4][8]; @@ -4451,7 +4451,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) kernel.packet[3] = pload(out[3]); } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { EIGEN_ALIGN16 float16x4x4_t tmp_x4; float16_t* tmp = (float16_t*)&kernel; tmp_x4 = vld4_f16(tmp); @@ -4462,7 +4462,7 @@ EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { kernel.packet[3] = tmp_x4.val[3]; } -EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { float16x8x2_t T_1[4]; T_1[0] = vuzpq_f16(kernel.packet[0], kernel.packet[1]); -- cgit v1.2.3