aboutsummaryrefslogtreecommitdiffhomepage
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/opts/SkNx_neon.h280
1 files changed, 182 insertions, 98 deletions
diff --git a/src/opts/SkNx_neon.h b/src/opts/SkNx_neon.h
index bf89b65534..e2574aeef0 100644
--- a/src/opts/SkNx_neon.h
+++ b/src/opts/SkNx_neon.h
@@ -17,9 +17,9 @@
// - subtract 1 if that's too big (possible for negative values).
// This restricts the domain of our inputs to a maximum somehwere around 2^31. Seems plenty big.
static inline float32x4_t armv7_vrndmq_f32(float32x4_t v) {
- float32x4_t roundtrip = vcvtq_f32_s32(vcvtq_s32_f32(v));
- uint32x4_t too_big = roundtrip > v;
- return roundtrip - (float32x4_t)vandq_u32(too_big, (uint32x4_t)vdupq_n_f32(1));
+ auto roundtrip = vcvtq_f32_s32(vcvtq_s32_f32(v));
+ auto too_big = vcgtq_f32(roundtrip, v);
+ return vsubq_f32(roundtrip, (float32x4_t)vandq_u32(too_big, (uint32x4_t)vdupq_n_f32(1)));
}
template <>
@@ -28,23 +28,40 @@ public:
SkNx(float32x2_t vec) : fVec(vec) {}
SkNx() {}
- SkNx(float a, float b) : fVec{a,b} {}
- SkNx(float v) : fVec{v,v} {}
-
+ SkNx(float val) : fVec(vdup_n_f32(val)) {}
static SkNx Load(const void* ptr) { return vld1_f32((const float*)ptr); }
+ SkNx(float a, float b) { fVec = (float32x2_t) { a, b }; }
+
void store(void* ptr) const { vst1_f32((float*)ptr, fVec); }
- SkNx operator + (const SkNx& o) const { return fVec + o.fVec; }
- SkNx operator - (const SkNx& o) const { return fVec - o.fVec; }
- SkNx operator * (const SkNx& o) const { return fVec * o.fVec; }
- SkNx operator / (const SkNx& o) const { return fVec / o.fVec; }
+ SkNx invert() const {
+ float32x2_t est0 = vrecpe_f32(fVec),
+ est1 = vmul_f32(vrecps_f32(est0, fVec), est0);
+ return est1;
+ }
+
+ SkNx operator + (const SkNx& o) const { return vadd_f32(fVec, o.fVec); }
+ SkNx operator - (const SkNx& o) const { return vsub_f32(fVec, o.fVec); }
+ SkNx operator * (const SkNx& o) const { return vmul_f32(fVec, o.fVec); }
+ SkNx operator / (const SkNx& o) const {
+ #if defined(SK_CPU_ARM64)
+ return vdiv_f32(fVec, o.fVec);
+ #else
+ float32x2_t est0 = vrecpe_f32(o.fVec),
+ est1 = vmul_f32(vrecps_f32(est0, o.fVec), est0),
+ est2 = vmul_f32(vrecps_f32(est1, o.fVec), est1);
+ return vmul_f32(fVec, est2);
+ #endif
+ }
- SkNx operator == (const SkNx& o) const { return fVec == o.fVec; }
- SkNx operator < (const SkNx& o) const { return fVec < o.fVec; }
- SkNx operator > (const SkNx& o) const { return fVec > o.fVec; }
- SkNx operator <= (const SkNx& o) const { return fVec <= o.fVec; }
- SkNx operator >= (const SkNx& o) const { return fVec >= o.fVec; }
- SkNx operator != (const SkNx& o) const { return fVec != o.fVec; }
+ SkNx operator == (const SkNx& o) const { return vreinterpret_f32_u32(vceq_f32(fVec, o.fVec)); }
+ SkNx operator < (const SkNx& o) const { return vreinterpret_f32_u32(vclt_f32(fVec, o.fVec)); }
+ SkNx operator > (const SkNx& o) const { return vreinterpret_f32_u32(vcgt_f32(fVec, o.fVec)); }
+ SkNx operator <= (const SkNx& o) const { return vreinterpret_f32_u32(vcle_f32(fVec, o.fVec)); }
+ SkNx operator >= (const SkNx& o) const { return vreinterpret_f32_u32(vcge_f32(fVec, o.fVec)); }
+ SkNx operator != (const SkNx& o) const {
+ return vreinterpret_f32_u32(vmvn_u32(vceq_f32(fVec, o.fVec)));
+ }
static SkNx Min(const SkNx& l, const SkNx& r) { return vmin_f32(l.fVec, r.fVec); }
static SkNx Max(const SkNx& l, const SkNx& r) { return vmax_f32(l.fVec, r.fVec); }
@@ -65,14 +82,12 @@ public:
#endif
}
- SkNx invert() const {
- float32x2_t est0 = vrecpe_f32(fVec),
- est1 = vmul_f32(vrecps_f32(est0, fVec), est0);
- return est1;
+ float operator[](int k) const {
+ SkASSERT(0 <= k && k < 2);
+ union { float32x2_t v; float fs[2]; } pun = {fVec};
+ return pun.fs[k&1];
}
- float operator[](int k) const { return fVec[k&1]; }
-
bool allTrue() const {
auto v = vreinterpret_u32_f32(fVec);
return vget_lane_u32(v,0) && vget_lane_u32(v,1);
@@ -91,23 +106,39 @@ public:
SkNx(float32x4_t vec) : fVec(vec) {}
SkNx() {}
- SkNx(float a, float b, float c, float d) : fVec{a,b,c,d} {}
- SkNx(float v) : fVec{v,v,v,v} {}
-
+ SkNx(float val) : fVec(vdupq_n_f32(val)) {}
static SkNx Load(const void* ptr) { return vld1q_f32((const float*)ptr); }
+ SkNx(float a, float b, float c, float d) { fVec = (float32x4_t) { a, b, c, d }; }
+
void store(void* ptr) const { vst1q_f32((float*)ptr, fVec); }
+ SkNx invert() const {
+ float32x4_t est0 = vrecpeq_f32(fVec),
+ est1 = vmulq_f32(vrecpsq_f32(est0, fVec), est0);
+ return est1;
+ }
- SkNx operator + (const SkNx& o) const { return fVec + o.fVec; }
- SkNx operator - (const SkNx& o) const { return fVec - o.fVec; }
- SkNx operator * (const SkNx& o) const { return fVec * o.fVec; }
- SkNx operator / (const SkNx& o) const { return fVec / o.fVec; }
+ SkNx operator + (const SkNx& o) const { return vaddq_f32(fVec, o.fVec); }
+ SkNx operator - (const SkNx& o) const { return vsubq_f32(fVec, o.fVec); }
+ SkNx operator * (const SkNx& o) const { return vmulq_f32(fVec, o.fVec); }
+ SkNx operator / (const SkNx& o) const {
+ #if defined(SK_CPU_ARM64)
+ return vdivq_f32(fVec, o.fVec);
+ #else
+ float32x4_t est0 = vrecpeq_f32(o.fVec),
+ est1 = vmulq_f32(vrecpsq_f32(est0, o.fVec), est0),
+ est2 = vmulq_f32(vrecpsq_f32(est1, o.fVec), est1);
+ return vmulq_f32(fVec, est2);
+ #endif
+ }
- SkNx operator==(const SkNx& o) const { return fVec == o.fVec; }
- SkNx operator <(const SkNx& o) const { return fVec < o.fVec; }
- SkNx operator >(const SkNx& o) const { return fVec > o.fVec; }
- SkNx operator<=(const SkNx& o) const { return fVec <= o.fVec; }
- SkNx operator>=(const SkNx& o) const { return fVec >= o.fVec; }
- SkNx operator!=(const SkNx& o) const { return fVec != o.fVec; }
+ SkNx operator==(const SkNx& o) const { return vreinterpretq_f32_u32(vceqq_f32(fVec, o.fVec)); }
+ SkNx operator <(const SkNx& o) const { return vreinterpretq_f32_u32(vcltq_f32(fVec, o.fVec)); }
+ SkNx operator >(const SkNx& o) const { return vreinterpretq_f32_u32(vcgtq_f32(fVec, o.fVec)); }
+ SkNx operator<=(const SkNx& o) const { return vreinterpretq_f32_u32(vcleq_f32(fVec, o.fVec)); }
+ SkNx operator>=(const SkNx& o) const { return vreinterpretq_f32_u32(vcgeq_f32(fVec, o.fVec)); }
+ SkNx operator!=(const SkNx& o) const {
+ return vreinterpretq_f32_u32(vmvnq_u32(vceqq_f32(fVec, o.fVec)));
+ }
static SkNx Min(const SkNx& l, const SkNx& r) { return vminq_f32(l.fVec, r.fVec); }
static SkNx Max(const SkNx& l, const SkNx& r) { return vmaxq_f32(l.fVec, r.fVec); }
@@ -121,6 +152,7 @@ public:
#endif
}
+
SkNx rsqrt() const {
float32x4_t est0 = vrsqrteq_f32(fVec);
return vmulq_f32(vrsqrtsq_f32(fVec, vmulq_f32(est0, est0)), est0);
@@ -137,14 +169,12 @@ public:
#endif
}
- SkNx invert() const {
- float32x4_t est0 = vrecpeq_f32(fVec),
- est1 = vmulq_f32(vrecpsq_f32(est0, fVec), est0);
- return est1;
+ float operator[](int k) const {
+ SkASSERT(0 <= k && k < 4);
+ union { float32x4_t v; float fs[4]; } pun = {fVec};
+ return pun.fs[k&3];
}
- float operator[](int k) const { return fVec[k&3]; }
-
bool allTrue() const {
auto v = vreinterpretq_u32_f32(fVec);
return vgetq_lane_u32(v,0) && vgetq_lane_u32(v,1)
@@ -172,22 +202,29 @@ public:
SkNx(const uint16x4_t& vec) : fVec(vec) {}
SkNx() {}
- SkNx(uint16_t a, uint16_t b, uint16_t c, uint16_t d) : fVec{a,b,c,d} {}
- SkNx(uint16_t v) : fVec{v,v,v,v} {}
-
+ SkNx(uint16_t val) : fVec(vdup_n_u16(val)) {}
static SkNx Load(const void* ptr) { return vld1_u16((const uint16_t*)ptr); }
+
+ SkNx(uint16_t a, uint16_t b, uint16_t c, uint16_t d) {
+ fVec = (uint16x4_t) { a,b,c,d };
+ }
+
void store(void* ptr) const { vst1_u16((uint16_t*)ptr, fVec); }
- SkNx operator + (const SkNx& o) const { return fVec + o.fVec; }
- SkNx operator - (const SkNx& o) const { return fVec - o.fVec; }
- SkNx operator * (const SkNx& o) const { return fVec * o.fVec; }
+ SkNx operator + (const SkNx& o) const { return vadd_u16(fVec, o.fVec); }
+ SkNx operator - (const SkNx& o) const { return vsub_u16(fVec, o.fVec); }
+ SkNx operator * (const SkNx& o) const { return vmul_u16(fVec, o.fVec); }
SkNx operator << (int bits) const { return fVec << SkNx(bits).fVec; }
SkNx operator >> (int bits) const { return fVec >> SkNx(bits).fVec; }
static SkNx Min(const SkNx& a, const SkNx& b) { return vmin_u16(a.fVec, b.fVec); }
- uint16_t operator[](int k) const { return fVec[k&3]; }
+ uint16_t operator[](int k) const {
+ SkASSERT(0 <= k && k < 4);
+ union { uint16x4_t v; uint16_t us[4]; } pun = {fVec};
+ return pun.us[k&3];
+ }
SkNx thenElse(const SkNx& t, const SkNx& e) const {
return vbsl_u16(fVec, t.fVec, e.fVec);
@@ -202,23 +239,30 @@ public:
SkNx(const uint16x8_t& vec) : fVec(vec) {}
SkNx() {}
+ SkNx(uint16_t val) : fVec(vdupq_n_u16(val)) {}
+ static SkNx Load(const void* ptr) { return vld1q_u16((const uint16_t*)ptr); }
+
SkNx(uint16_t a, uint16_t b, uint16_t c, uint16_t d,
- uint16_t e, uint16_t f, uint16_t g, uint16_t h) : fVec{a,b,c,d,e,f,g,h} {}
- SkNx(uint16_t v) : fVec{v,v,v,v,v,v,v,v} {}
+ uint16_t e, uint16_t f, uint16_t g, uint16_t h) {
+ fVec = (uint16x8_t) { a,b,c,d, e,f,g,h };
+ }
- static SkNx Load(const void* ptr) { return vld1q_u16((const uint16_t*)ptr); }
void store(void* ptr) const { vst1q_u16((uint16_t*)ptr, fVec); }
- SkNx operator + (const SkNx& o) const { return fVec + o.fVec; }
- SkNx operator - (const SkNx& o) const { return fVec - o.fVec; }
- SkNx operator * (const SkNx& o) const { return fVec * o.fVec; }
+ SkNx operator + (const SkNx& o) const { return vaddq_u16(fVec, o.fVec); }
+ SkNx operator - (const SkNx& o) const { return vsubq_u16(fVec, o.fVec); }
+ SkNx operator * (const SkNx& o) const { return vmulq_u16(fVec, o.fVec); }
SkNx operator << (int bits) const { return fVec << SkNx(bits).fVec; }
SkNx operator >> (int bits) const { return fVec >> SkNx(bits).fVec; }
static SkNx Min(const SkNx& a, const SkNx& b) { return vminq_u16(a.fVec, b.fVec); }
- uint16_t operator[](int k) const { return fVec[k&7]; }
+ uint16_t operator[](int k) const {
+ SkASSERT(0 <= k && k < 8);
+ union { uint16x8_t v; uint16_t us[8]; } pun = {fVec};
+ return pun.us[k&7];
+ }
SkNx thenElse(const SkNx& t, const SkNx& e) const {
return vbslq_u16(fVec, t.fVec, e.fVec);
@@ -235,17 +279,22 @@ public:
SkNx(const uint8x8_t& vec) : fVec(vec) {}
SkNx() {}
- SkNx(uint8_t a, uint8_t b, uint8_t c, uint8_t d) : fVec{a,b,c,d,0,0,0,0} {}
- SkNx(uint8_t v) : fVec{v,v,v,v,0,0,0,0} {}
-
+ SkNx(uint8_t a, uint8_t b, uint8_t c, uint8_t d) {
+ fVec = (uint8x8_t){a,b,c,d, 0,0,0,0};
+ }
static SkNx Load(const void* ptr) {
return (uint8x8_t)vld1_dup_u32((const unaligned_uint32_t*)ptr);
}
void store(void* ptr) const {
return vst1_lane_u32((unaligned_uint32_t*)ptr, (uint32x2_t)fVec, 0);
}
+ uint8_t operator[](int k) const {
+ SkASSERT(0 <= k && k < 4);
+ union { uint8x8_t v; uint8_t us[8]; } pun = {fVec};
+ return pun.us[k&3];
+ }
- uint8_t operator[](int k) const { return fVec[k&3]; }
+ // TODO as needed
uint8x8_t fVec;
};
@@ -256,24 +305,31 @@ public:
SkNx(const uint8x16_t& vec) : fVec(vec) {}
SkNx() {}
+ SkNx(uint8_t val) : fVec(vdupq_n_u8(val)) {}
+ static SkNx Load(const void* ptr) { return vld1q_u8((const uint8_t*)ptr); }
+
SkNx(uint8_t a, uint8_t b, uint8_t c, uint8_t d,
uint8_t e, uint8_t f, uint8_t g, uint8_t h,
uint8_t i, uint8_t j, uint8_t k, uint8_t l,
- uint8_t m, uint8_t n, uint8_t o, uint8_t p) : fVec{a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p} {}
- SkNx(uint8_t v) : fVec{v,v,v,v,v,v,v,v,v,v,v,v,v,v,v,v} {}
+ uint8_t m, uint8_t n, uint8_t o, uint8_t p) {
+ fVec = (uint8x16_t) { a,b,c,d, e,f,g,h, i,j,k,l, m,n,o,p };
+ }
- static SkNx Load(const void* ptr) { return vld1q_u8((const uint8_t*)ptr); }
void store(void* ptr) const { vst1q_u8((uint8_t*)ptr, fVec); }
SkNx saturatedAdd(const SkNx& o) const { return vqaddq_u8(fVec, o.fVec); }
- SkNx operator + (const SkNx& o) const { return fVec + o.fVec; }
- SkNx operator - (const SkNx& o) const { return fVec - o.fVec; }
- SkNx operator < (const SkNx& o) const { return fVec < o.fVec; }
+ SkNx operator + (const SkNx& o) const { return vaddq_u8(fVec, o.fVec); }
+ SkNx operator - (const SkNx& o) const { return vsubq_u8(fVec, o.fVec); }
static SkNx Min(const SkNx& a, const SkNx& b) { return vminq_u8(a.fVec, b.fVec); }
+ SkNx operator < (const SkNx& o) const { return vcltq_u8(fVec, o.fVec); }
- uint8_t operator[](int k) const { return fVec[k&15]; }
+ uint8_t operator[](int k) const {
+ SkASSERT(0 <= k && k < 16);
+ union { uint8x16_t v; uint8_t us[16]; } pun = {fVec};
+ return pun.us[k&15];
+ }
SkNx thenElse(const SkNx& t, const SkNx& e) const {
return vbslq_u8(fVec, t.fVec, e.fVec);
@@ -288,30 +344,47 @@ public:
SkNx(const int32x4_t& vec) : fVec(vec) {}
SkNx() {}
- SkNx(int32_t a, int32_t b, int32_t c, int32_t d) : fVec{a,b,c,d} {}
- SkNx(int32_t v) : fVec{v,v,v,v} {}
-
- static SkNx Load(const void* ptr) { return vld1q_s32((const int32_t*)ptr); }
- void store(void* ptr) const { return vst1q_s32((int32_t*)ptr, fVec); }
+ SkNx(int32_t v) {
+ fVec = vdupq_n_s32(v);
+ }
+ SkNx(int32_t a, int32_t b, int32_t c, int32_t d) {
+ fVec = (int32x4_t){a,b,c,d};
+ }
+ static SkNx Load(const void* ptr) {
+ return vld1q_s32((const int32_t*)ptr);
+ }
+ void store(void* ptr) const {
+ return vst1q_s32((int32_t*)ptr, fVec);
+ }
+ int32_t operator[](int k) const {
+ SkASSERT(0 <= k && k < 4);
+ union { int32x4_t v; int32_t is[4]; } pun = {fVec};
+ return pun.is[k&3];
+ }
- SkNx operator + (const SkNx& o) const { return fVec + o.fVec; }
- SkNx operator - (const SkNx& o) const { return fVec - o.fVec; }
- SkNx operator * (const SkNx& o) const { return fVec * o.fVec; }
+ SkNx operator + (const SkNx& o) const { return vaddq_s32(fVec, o.fVec); }
+ SkNx operator - (const SkNx& o) const { return vsubq_s32(fVec, o.fVec); }
+ SkNx operator * (const SkNx& o) const { return vmulq_s32(fVec, o.fVec); }
- SkNx operator & (const SkNx& o) const { return fVec & o.fVec; }
- SkNx operator | (const SkNx& o) const { return fVec | o.fVec; }
- SkNx operator ^ (const SkNx& o) const { return fVec ^ o.fVec; }
+ SkNx operator & (const SkNx& o) const { return vandq_s32(fVec, o.fVec); }
+ SkNx operator | (const SkNx& o) const { return vorrq_s32(fVec, o.fVec); }
+ SkNx operator ^ (const SkNx& o) const { return veorq_s32(fVec, o.fVec); }
SkNx operator << (int bits) const { return fVec << SkNx(bits).fVec; }
SkNx operator >> (int bits) const { return fVec >> SkNx(bits).fVec; }
- SkNx operator == (const SkNx& o) const { return fVec == o.fVec; }
- SkNx operator < (const SkNx& o) const { return fVec < o.fVec; }
- SkNx operator > (const SkNx& o) const { return fVec > o.fVec; }
+ SkNx operator == (const SkNx& o) const {
+ return vreinterpretq_s32_u32(vceqq_s32(fVec, o.fVec));
+ }
+ SkNx operator < (const SkNx& o) const {
+ return vreinterpretq_s32_u32(vcltq_s32(fVec, o.fVec));
+ }
+ SkNx operator > (const SkNx& o) const {
+ return vreinterpretq_s32_u32(vcgtq_s32(fVec, o.fVec));
+ }
static SkNx Min(const SkNx& a, const SkNx& b) { return vminq_s32(a.fVec, b.fVec); }
-
- int32_t operator[](int k) const { return fVec[k&3]; }
+ // TODO as needed
SkNx thenElse(const SkNx& t, const SkNx& e) const {
return vbslq_s32(vreinterpretq_u32_s32(fVec), t.fVec, e.fVec);
@@ -326,30 +399,41 @@ public:
SkNx(const uint32x4_t& vec) : fVec(vec) {}
SkNx() {}
- SkNx(uint32_t a, uint32_t b, uint32_t c, uint32_t d) : fVec{a,b,c,d} {}
- SkNx(uint32_t v) : fVec{v,v,v,v} {}
-
- static SkNx Load(const void* ptr) { return vld1q_u32((const uint32_t*)ptr); }
- void store(void* ptr) const { return vst1q_u32((uint32_t*)ptr, fVec); }
+ SkNx(uint32_t v) {
+ fVec = vdupq_n_u32(v);
+ }
+ SkNx(uint32_t a, uint32_t b, uint32_t c, uint32_t d) {
+ fVec = (uint32x4_t){a,b,c,d};
+ }
+ static SkNx Load(const void* ptr) {
+ return vld1q_u32((const uint32_t*)ptr);
+ }
+ void store(void* ptr) const {
+ return vst1q_u32((uint32_t*)ptr, fVec);
+ }
+ uint32_t operator[](int k) const {
+ SkASSERT(0 <= k && k < 4);
+ union { uint32x4_t v; uint32_t us[4]; } pun = {fVec};
+ return pun.us[k&3];
+ }
- SkNx operator + (const SkNx& o) const { return fVec + o.fVec; }
- SkNx operator - (const SkNx& o) const { return fVec - o.fVec; }
- SkNx operator * (const SkNx& o) const { return fVec * o.fVec; }
+ SkNx operator + (const SkNx& o) const { return vaddq_u32(fVec, o.fVec); }
+ SkNx operator - (const SkNx& o) const { return vsubq_u32(fVec, o.fVec); }
+ SkNx operator * (const SkNx& o) const { return vmulq_u32(fVec, o.fVec); }
- SkNx operator & (const SkNx& o) const { return fVec & o.fVec; }
- SkNx operator | (const SkNx& o) const { return fVec | o.fVec; }
- SkNx operator ^ (const SkNx& o) const { return fVec ^ o.fVec; }
+ SkNx operator & (const SkNx& o) const { return vandq_u32(fVec, o.fVec); }
+ SkNx operator | (const SkNx& o) const { return vorrq_u32(fVec, o.fVec); }
+ SkNx operator ^ (const SkNx& o) const { return veorq_u32(fVec, o.fVec); }
SkNx operator << (int bits) const { return fVec << SkNx(bits).fVec; }
SkNx operator >> (int bits) const { return fVec >> SkNx(bits).fVec; }
- SkNx operator == (const SkNx& o) const { return fVec == o.fVec; }
- SkNx operator < (const SkNx& o) const { return fVec < o.fVec; }
- SkNx operator > (const SkNx& o) const { return fVec > o.fVec; }
+ SkNx operator == (const SkNx& o) const { return vceqq_u32(fVec, o.fVec); }
+ SkNx operator < (const SkNx& o) const { return vcltq_u32(fVec, o.fVec); }
+ SkNx operator > (const SkNx& o) const { return vcgtq_u32(fVec, o.fVec); }
static SkNx Min(const SkNx& a, const SkNx& b) { return vminq_u32(a.fVec, b.fVec); }
-
- uint32_t operator[](int k) const { return fVec[k&3]; }
+ // TODO as needed
SkNx thenElse(const SkNx& t, const SkNx& e) const {
return vbslq_u32(fVec, t.fVec, e.fVec);
generated by cgit v1.2.3 (git 2.39.1) at 2024-08-28 17:48:37 +0000