diff options
| -rw-r--r-- | src/opts/SkNx_neon.h | 280 |
1 files changed, 182 insertions, 98 deletions
diff --git a/src/opts/SkNx_neon.h b/src/opts/SkNx_neon.h index bf89b65534..e2574aeef0 100644 --- a/src/opts/SkNx_neon.h +++ b/src/opts/SkNx_neon.h @@ -17,9 +17,9 @@ // - subtract 1 if that's too big (possible for negative values). // This restricts the domain of our inputs to a maximum somehwere around 2^31. Seems plenty big. static inline float32x4_t armv7_vrndmq_f32(float32x4_t v) { - float32x4_t roundtrip = vcvtq_f32_s32(vcvtq_s32_f32(v)); - uint32x4_t too_big = roundtrip > v; - return roundtrip - (float32x4_t)vandq_u32(too_big, (uint32x4_t)vdupq_n_f32(1)); + auto roundtrip = vcvtq_f32_s32(vcvtq_s32_f32(v)); + auto too_big = vcgtq_f32(roundtrip, v); + return vsubq_f32(roundtrip, (float32x4_t)vandq_u32(too_big, (uint32x4_t)vdupq_n_f32(1))); } template <> @@ -28,23 +28,40 @@ public: SkNx(float32x2_t vec) : fVec(vec) {} SkNx() {} - SkNx(float a, float b) : fVec{a,b} {} - SkNx(float v) : fVec{v,v} {} - + SkNx(float val) : fVec(vdup_n_f32(val)) {} static SkNx Load(const void* ptr) { return vld1_f32((const float*)ptr); } + SkNx(float a, float b) { fVec = (float32x2_t) { a, b }; } + void store(void* ptr) const { vst1_f32((float*)ptr, fVec); } - SkNx operator + (const SkNx& o) const { return fVec + o.fVec; } - SkNx operator - (const SkNx& o) const { return fVec - o.fVec; } - SkNx operator * (const SkNx& o) const { return fVec * o.fVec; } - SkNx operator / (const SkNx& o) const { return fVec / o.fVec; } + SkNx invert() const { + float32x2_t est0 = vrecpe_f32(fVec), + est1 = vmul_f32(vrecps_f32(est0, fVec), est0); + return est1; + } + + SkNx operator + (const SkNx& o) const { return vadd_f32(fVec, o.fVec); } + SkNx operator - (const SkNx& o) const { return vsub_f32(fVec, o.fVec); } + SkNx operator * (const SkNx& o) const { return vmul_f32(fVec, o.fVec); } + SkNx operator / (const SkNx& o) const { + #if defined(SK_CPU_ARM64) + return vdiv_f32(fVec, o.fVec); + #else + float32x2_t est0 = vrecpe_f32(o.fVec), + est1 = vmul_f32(vrecps_f32(est0, o.fVec), est0), + est2 = vmul_f32(vrecps_f32(est1, o.fVec), est1); + return vmul_f32(fVec, est2); + #endif + } - SkNx operator == (const SkNx& o) const { return fVec == o.fVec; } - SkNx operator < (const SkNx& o) const { return fVec < o.fVec; } - SkNx operator > (const SkNx& o) const { return fVec > o.fVec; } - SkNx operator <= (const SkNx& o) const { return fVec <= o.fVec; } - SkNx operator >= (const SkNx& o) const { return fVec >= o.fVec; } - SkNx operator != (const SkNx& o) const { return fVec != o.fVec; } + SkNx operator == (const SkNx& o) const { return vreinterpret_f32_u32(vceq_f32(fVec, o.fVec)); } + SkNx operator < (const SkNx& o) const { return vreinterpret_f32_u32(vclt_f32(fVec, o.fVec)); } + SkNx operator > (const SkNx& o) const { return vreinterpret_f32_u32(vcgt_f32(fVec, o.fVec)); } + SkNx operator <= (const SkNx& o) const { return vreinterpret_f32_u32(vcle_f32(fVec, o.fVec)); } + SkNx operator >= (const SkNx& o) const { return vreinterpret_f32_u32(vcge_f32(fVec, o.fVec)); } + SkNx operator != (const SkNx& o) const { + return vreinterpret_f32_u32(vmvn_u32(vceq_f32(fVec, o.fVec))); + } static SkNx Min(const SkNx& l, const SkNx& r) { return vmin_f32(l.fVec, r.fVec); } static SkNx Max(const SkNx& l, const SkNx& r) { return vmax_f32(l.fVec, r.fVec); } @@ -65,14 +82,12 @@ public: #endif } - SkNx invert() const { - float32x2_t est0 = vrecpe_f32(fVec), - est1 = vmul_f32(vrecps_f32(est0, fVec), est0); - return est1; + float operator[](int k) const { + SkASSERT(0 <= k && k < 2); + union { float32x2_t v; float fs[2]; } pun = {fVec}; + return pun.fs[k&1]; } - float operator[](int k) const { return fVec[k&1]; } - bool allTrue() const { auto v = vreinterpret_u32_f32(fVec); return vget_lane_u32(v,0) && vget_lane_u32(v,1); @@ -91,23 +106,39 @@ public: SkNx(float32x4_t vec) : fVec(vec) {} SkNx() {} - SkNx(float a, float b, float c, float d) : fVec{a,b,c,d} {} - SkNx(float v) : fVec{v,v,v,v} {} - + SkNx(float val) : fVec(vdupq_n_f32(val)) {} static SkNx Load(const void* ptr) { return vld1q_f32((const float*)ptr); } + SkNx(float a, float b, float c, float d) { fVec = (float32x4_t) { a, b, c, d }; } + void store(void* ptr) const { vst1q_f32((float*)ptr, fVec); } + SkNx invert() const { + float32x4_t est0 = vrecpeq_f32(fVec), + est1 = vmulq_f32(vrecpsq_f32(est0, fVec), est0); + return est1; + } - SkNx operator + (const SkNx& o) const { return fVec + o.fVec; } - SkNx operator - (const SkNx& o) const { return fVec - o.fVec; } - SkNx operator * (const SkNx& o) const { return fVec * o.fVec; } - SkNx operator / (const SkNx& o) const { return fVec / o.fVec; } + SkNx operator + (const SkNx& o) const { return vaddq_f32(fVec, o.fVec); } + SkNx operator - (const SkNx& o) const { return vsubq_f32(fVec, o.fVec); } + SkNx operator * (const SkNx& o) const { return vmulq_f32(fVec, o.fVec); } + SkNx operator / (const SkNx& o) const { + #if defined(SK_CPU_ARM64) + return vdivq_f32(fVec, o.fVec); + #else + float32x4_t est0 = vrecpeq_f32(o.fVec), + est1 = vmulq_f32(vrecpsq_f32(est0, o.fVec), est0), + est2 = vmulq_f32(vrecpsq_f32(est1, o.fVec), est1); + return vmulq_f32(fVec, est2); + #endif + } - SkNx operator==(const SkNx& o) const { return fVec == o.fVec; } - SkNx operator <(const SkNx& o) const { return fVec < o.fVec; } - SkNx operator >(const SkNx& o) const { return fVec > o.fVec; } - SkNx operator<=(const SkNx& o) const { return fVec <= o.fVec; } - SkNx operator>=(const SkNx& o) const { return fVec >= o.fVec; } - SkNx operator!=(const SkNx& o) const { return fVec != o.fVec; } + SkNx operator==(const SkNx& o) const { return vreinterpretq_f32_u32(vceqq_f32(fVec, o.fVec)); } + SkNx operator <(const SkNx& o) const { return vreinterpretq_f32_u32(vcltq_f32(fVec, o.fVec)); } + SkNx operator >(const SkNx& o) const { return vreinterpretq_f32_u32(vcgtq_f32(fVec, o.fVec)); } + SkNx operator<=(const SkNx& o) const { return vreinterpretq_f32_u32(vcleq_f32(fVec, o.fVec)); } + SkNx operator>=(const SkNx& o) const { return vreinterpretq_f32_u32(vcgeq_f32(fVec, o.fVec)); } + SkNx operator!=(const SkNx& o) const { + return vreinterpretq_f32_u32(vmvnq_u32(vceqq_f32(fVec, o.fVec))); + } static SkNx Min(const SkNx& l, const SkNx& r) { return vminq_f32(l.fVec, r.fVec); } static SkNx Max(const SkNx& l, const SkNx& r) { return vmaxq_f32(l.fVec, r.fVec); } @@ -121,6 +152,7 @@ public: #endif } + SkNx rsqrt() const { float32x4_t est0 = vrsqrteq_f32(fVec); return vmulq_f32(vrsqrtsq_f32(fVec, vmulq_f32(est0, est0)), est0); @@ -137,14 +169,12 @@ public: #endif } - SkNx invert() const { - float32x4_t est0 = vrecpeq_f32(fVec), - est1 = vmulq_f32(vrecpsq_f32(est0, fVec), est0); - return est1; + float operator[](int k) const { + SkASSERT(0 <= k && k < 4); + union { float32x4_t v; float fs[4]; } pun = {fVec}; + return pun.fs[k&3]; } - float operator[](int k) const { return fVec[k&3]; } - bool allTrue() const { auto v = vreinterpretq_u32_f32(fVec); return vgetq_lane_u32(v,0) && vgetq_lane_u32(v,1) @@ -172,22 +202,29 @@ public: SkNx(const uint16x4_t& vec) : fVec(vec) {} SkNx() {} - SkNx(uint16_t a, uint16_t b, uint16_t c, uint16_t d) : fVec{a,b,c,d} {} - SkNx(uint16_t v) : fVec{v,v,v,v} {} - + SkNx(uint16_t val) : fVec(vdup_n_u16(val)) {} static SkNx Load(const void* ptr) { return vld1_u16((const uint16_t*)ptr); } + + SkNx(uint16_t a, uint16_t b, uint16_t c, uint16_t d) { + fVec = (uint16x4_t) { a,b,c,d }; + } + void store(void* ptr) const { vst1_u16((uint16_t*)ptr, fVec); } - SkNx operator + (const SkNx& o) const { return fVec + o.fVec; } - SkNx operator - (const SkNx& o) const { return fVec - o.fVec; } - SkNx operator * (const SkNx& o) const { return fVec * o.fVec; } + SkNx operator + (const SkNx& o) const { return vadd_u16(fVec, o.fVec); } + SkNx operator - (const SkNx& o) const { return vsub_u16(fVec, o.fVec); } + SkNx operator * (const SkNx& o) const { return vmul_u16(fVec, o.fVec); } SkNx operator << (int bits) const { return fVec << SkNx(bits).fVec; } SkNx operator >> (int bits) const { return fVec >> SkNx(bits).fVec; } static SkNx Min(const SkNx& a, const SkNx& b) { return vmin_u16(a.fVec, b.fVec); } - uint16_t operator[](int k) const { return fVec[k&3]; } + uint16_t operator[](int k) const { + SkASSERT(0 <= k && k < 4); + union { uint16x4_t v; uint16_t us[4]; } pun = {fVec}; + return pun.us[k&3]; + } SkNx thenElse(const SkNx& t, const SkNx& e) const { return vbsl_u16(fVec, t.fVec, e.fVec); @@ -202,23 +239,30 @@ public: SkNx(const uint16x8_t& vec) : fVec(vec) {} SkNx() {} + SkNx(uint16_t val) : fVec(vdupq_n_u16(val)) {} + static SkNx Load(const void* ptr) { return vld1q_u16((const uint16_t*)ptr); } + SkNx(uint16_t a, uint16_t b, uint16_t c, uint16_t d, - uint16_t e, uint16_t f, uint16_t g, uint16_t h) : fVec{a,b,c,d,e,f,g,h} {} - SkNx(uint16_t v) : fVec{v,v,v,v,v,v,v,v} {} + uint16_t e, uint16_t f, uint16_t g, uint16_t h) { + fVec = (uint16x8_t) { a,b,c,d, e,f,g,h }; + } - static SkNx Load(const void* ptr) { return vld1q_u16((const uint16_t*)ptr); } void store(void* ptr) const { vst1q_u16((uint16_t*)ptr, fVec); } - SkNx operator + (const SkNx& o) const { return fVec + o.fVec; } - SkNx operator - (const SkNx& o) const { return fVec - o.fVec; } - SkNx operator * (const SkNx& o) const { return fVec * o.fVec; } + SkNx operator + (const SkNx& o) const { return vaddq_u16(fVec, o.fVec); } + SkNx operator - (const SkNx& o) const { return vsubq_u16(fVec, o.fVec); } + SkNx operator * (const SkNx& o) const { return vmulq_u16(fVec, o.fVec); } SkNx operator << (int bits) const { return fVec << SkNx(bits).fVec; } SkNx operator >> (int bits) const { return fVec >> SkNx(bits).fVec; } static SkNx Min(const SkNx& a, const SkNx& b) { return vminq_u16(a.fVec, b.fVec); } - uint16_t operator[](int k) const { return fVec[k&7]; } + uint16_t operator[](int k) const { + SkASSERT(0 <= k && k < 8); + union { uint16x8_t v; uint16_t us[8]; } pun = {fVec}; + return pun.us[k&7]; + } SkNx thenElse(const SkNx& t, const SkNx& e) const { return vbslq_u16(fVec, t.fVec, e.fVec); @@ -235,17 +279,22 @@ public: SkNx(const uint8x8_t& vec) : fVec(vec) {} SkNx() {} - SkNx(uint8_t a, uint8_t b, uint8_t c, uint8_t d) : fVec{a,b,c,d,0,0,0,0} {} - SkNx(uint8_t v) : fVec{v,v,v,v,0,0,0,0} {} - + SkNx(uint8_t a, uint8_t b, uint8_t c, uint8_t d) { + fVec = (uint8x8_t){a,b,c,d, 0,0,0,0}; + } static SkNx Load(const void* ptr) { return (uint8x8_t)vld1_dup_u32((const unaligned_uint32_t*)ptr); } void store(void* ptr) const { return vst1_lane_u32((unaligned_uint32_t*)ptr, (uint32x2_t)fVec, 0); } + uint8_t operator[](int k) const { + SkASSERT(0 <= k && k < 4); + union { uint8x8_t v; uint8_t us[8]; } pun = {fVec}; + return pun.us[k&3]; + } - uint8_t operator[](int k) const { return fVec[k&3]; } + // TODO as needed uint8x8_t fVec; }; @@ -256,24 +305,31 @@ public: SkNx(const uint8x16_t& vec) : fVec(vec) {} SkNx() {} + SkNx(uint8_t val) : fVec(vdupq_n_u8(val)) {} + static SkNx Load(const void* ptr) { return vld1q_u8((const uint8_t*)ptr); } + SkNx(uint8_t a, uint8_t b, uint8_t c, uint8_t d, uint8_t e, uint8_t f, uint8_t g, uint8_t h, uint8_t i, uint8_t j, uint8_t k, uint8_t l, - uint8_t m, uint8_t n, uint8_t o, uint8_t p) : fVec{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p} {} - SkNx(uint8_t v) : fVec{v,v,v,v,v,v,v,v,v,v,v,v,v,v,v,v} {} + uint8_t m, uint8_t n, uint8_t o, uint8_t p) { + fVec = (uint8x16_t) { a,b,c,d, e,f,g,h, i,j,k,l, m,n,o,p }; + } - static SkNx Load(const void* ptr) { return vld1q_u8((const uint8_t*)ptr); } void store(void* ptr) const { vst1q_u8((uint8_t*)ptr, fVec); } SkNx saturatedAdd(const SkNx& o) const { return vqaddq_u8(fVec, o.fVec); } - SkNx operator + (const SkNx& o) const { return fVec + o.fVec; } - SkNx operator - (const SkNx& o) const { return fVec - o.fVec; } - SkNx operator < (const SkNx& o) const { return fVec < o.fVec; } + SkNx operator + (const SkNx& o) const { return vaddq_u8(fVec, o.fVec); } + SkNx operator - (const SkNx& o) const { return vsubq_u8(fVec, o.fVec); } static SkNx Min(const SkNx& a, const SkNx& b) { return vminq_u8(a.fVec, b.fVec); } + SkNx operator < (const SkNx& o) const { return vcltq_u8(fVec, o.fVec); } - uint8_t operator[](int k) const { return fVec[k&15]; } + uint8_t operator[](int k) const { + SkASSERT(0 <= k && k < 16); + union { uint8x16_t v; uint8_t us[16]; } pun = {fVec}; + return pun.us[k&15]; + } SkNx thenElse(const SkNx& t, const SkNx& e) const { return vbslq_u8(fVec, t.fVec, e.fVec); @@ -288,30 +344,47 @@ public: SkNx(const int32x4_t& vec) : fVec(vec) {} SkNx() {} - SkNx(int32_t a, int32_t b, int32_t c, int32_t d) : fVec{a,b,c,d} {} - SkNx(int32_t v) : fVec{v,v,v,v} {} - - static SkNx Load(const void* ptr) { return vld1q_s32((const int32_t*)ptr); } - void store(void* ptr) const { return vst1q_s32((int32_t*)ptr, fVec); } + SkNx(int32_t v) { + fVec = vdupq_n_s32(v); + } + SkNx(int32_t a, int32_t b, int32_t c, int32_t d) { + fVec = (int32x4_t){a,b,c,d}; + } + static SkNx Load(const void* ptr) { + return vld1q_s32((const int32_t*)ptr); + } + void store(void* ptr) const { + return vst1q_s32((int32_t*)ptr, fVec); + } + int32_t operator[](int k) const { + SkASSERT(0 <= k && k < 4); + union { int32x4_t v; int32_t is[4]; } pun = {fVec}; + return pun.is[k&3]; + } - SkNx operator + (const SkNx& o) const { return fVec + o.fVec; } - SkNx operator - (const SkNx& o) const { return fVec - o.fVec; } - SkNx operator * (const SkNx& o) const { return fVec * o.fVec; } + SkNx operator + (const SkNx& o) const { return vaddq_s32(fVec, o.fVec); } + SkNx operator - (const SkNx& o) const { return vsubq_s32(fVec, o.fVec); } + SkNx operator * (const SkNx& o) const { return vmulq_s32(fVec, o.fVec); } - SkNx operator & (const SkNx& o) const { return fVec & o.fVec; } - SkNx operator | (const SkNx& o) const { return fVec | o.fVec; } - SkNx operator ^ (const SkNx& o) const { return fVec ^ o.fVec; } + SkNx operator & (const SkNx& o) const { return vandq_s32(fVec, o.fVec); } + SkNx operator | (const SkNx& o) const { return vorrq_s32(fVec, o.fVec); } + SkNx operator ^ (const SkNx& o) const { return veorq_s32(fVec, o.fVec); } SkNx operator << (int bits) const { return fVec << SkNx(bits).fVec; } SkNx operator >> (int bits) const { return fVec >> SkNx(bits).fVec; } - SkNx operator == (const SkNx& o) const { return fVec == o.fVec; } - SkNx operator < (const SkNx& o) const { return fVec < o.fVec; } - SkNx operator > (const SkNx& o) const { return fVec > o.fVec; } + SkNx operator == (const SkNx& o) const { + return vreinterpretq_s32_u32(vceqq_s32(fVec, o.fVec)); + } + SkNx operator < (const SkNx& o) const { + return vreinterpretq_s32_u32(vcltq_s32(fVec, o.fVec)); + } + SkNx operator > (const SkNx& o) const { + return vreinterpretq_s32_u32(vcgtq_s32(fVec, o.fVec)); + } static SkNx Min(const SkNx& a, const SkNx& b) { return vminq_s32(a.fVec, b.fVec); } - - int32_t operator[](int k) const { return fVec[k&3]; } + // TODO as needed SkNx thenElse(const SkNx& t, const SkNx& e) const { return vbslq_s32(vreinterpretq_u32_s32(fVec), t.fVec, e.fVec); @@ -326,30 +399,41 @@ public: SkNx(const uint32x4_t& vec) : fVec(vec) {} SkNx() {} - SkNx(uint32_t a, uint32_t b, uint32_t c, uint32_t d) : fVec{a,b,c,d} {} - SkNx(uint32_t v) : fVec{v,v,v,v} {} - - static SkNx Load(const void* ptr) { return vld1q_u32((const uint32_t*)ptr); } - void store(void* ptr) const { return vst1q_u32((uint32_t*)ptr, fVec); } + SkNx(uint32_t v) { + fVec = vdupq_n_u32(v); + } + SkNx(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { + fVec = (uint32x4_t){a,b,c,d}; + } + static SkNx Load(const void* ptr) { + return vld1q_u32((const uint32_t*)ptr); + } + void store(void* ptr) const { + return vst1q_u32((uint32_t*)ptr, fVec); + } + uint32_t operator[](int k) const { + SkASSERT(0 <= k && k < 4); + union { uint32x4_t v; uint32_t us[4]; } pun = {fVec}; + return pun.us[k&3]; + } - SkNx operator + (const SkNx& o) const { return fVec + o.fVec; } - SkNx operator - (const SkNx& o) const { return fVec - o.fVec; } - SkNx operator * (const SkNx& o) const { return fVec * o.fVec; } + SkNx operator + (const SkNx& o) const { return vaddq_u32(fVec, o.fVec); } + SkNx operator - (const SkNx& o) const { return vsubq_u32(fVec, o.fVec); } + SkNx operator * (const SkNx& o) const { return vmulq_u32(fVec, o.fVec); } - SkNx operator & (const SkNx& o) const { return fVec & o.fVec; } - SkNx operator | (const SkNx& o) const { return fVec | o.fVec; } - SkNx operator ^ (const SkNx& o) const { return fVec ^ o.fVec; } + SkNx operator & (const SkNx& o) const { return vandq_u32(fVec, o.fVec); } + SkNx operator | (const SkNx& o) const { return vorrq_u32(fVec, o.fVec); } + SkNx operator ^ (const SkNx& o) const { return veorq_u32(fVec, o.fVec); } SkNx operator << (int bits) const { return fVec << SkNx(bits).fVec; } SkNx operator >> (int bits) const { return fVec >> SkNx(bits).fVec; } - SkNx operator == (const SkNx& o) const { return fVec == o.fVec; } - SkNx operator < (const SkNx& o) const { return fVec < o.fVec; } - SkNx operator > (const SkNx& o) const { return fVec > o.fVec; } + SkNx operator == (const SkNx& o) const { return vceqq_u32(fVec, o.fVec); } + SkNx operator < (const SkNx& o) const { return vcltq_u32(fVec, o.fVec); } + SkNx operator > (const SkNx& o) const { return vcgtq_u32(fVec, o.fVec); } static SkNx Min(const SkNx& a, const SkNx& b) { return vminq_u32(a.fVec, b.fVec); } - - uint32_t operator[](int k) const { return fVec[k&3]; } + // TODO as needed SkNx thenElse(const SkNx& t, const SkNx& e) const { return vbslq_u32(fVec, t.fVec, e.fVec); |