Specialize Sk2d for ARM64

The implementation is nearly identical to Sk2f, with these changes:
  - float32x2_t -> float64x2_t
  - vfoo -> vfooq
  - one extra Newton's method step in sqrt().

Also, generally fix NEON detection to be defined(SK_ARM_HAS_NEON).
SK_ARM_HAS_NEON is not being set on ARM64 bots right now (nor does the compiler
seem to set __ARM_NEON__), so this CL fixes everything up.

BUG=skia:

Committed: https://skia.googlesource.com/skia/+/e57b5cab261a243dcbefa74c91c896c28959bf09

CQ_EXTRA_TRYBOTS=client.skia.compile:Build-Mac10.7-Clang-Arm7-Debug-iOS-Trybot,Build-Ubuntu-GCC-Arm64-Release-Android-Trybot

Review URL: https://codereview.chromium.org/1020963002

5 files changed, 72 insertions, 76 deletions

diff --git a/src/core/Sk2x.h b/src/core/Sk2x.h
index 3fbe76c37f..a64ad721a5 100644
--- a/src/core/Sk2x.h
+++ b/src/core/Sk2x.h
@@ -14,7 +14,7 @@
 #define SK2X_PREAMBLE 1
     #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE2 && !defined(SKNX_NO_SIMD)
         #include "../opts/Sk2x_sse.h"
-    #elif defined(__ARM_NEON__)                   && !defined(SKNX_NO_SIMD)
+    #elif defined(SK_ARM_HAS_NEON)                && !defined(SKNX_NO_SIMD)
         #include "../opts/Sk2x_neon.h"
     #else
         #include "../opts/Sk2x_none.h"
@@ -71,7 +71,7 @@ private:
 #define SK2X_PRIVATE 1
     #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE2 && !defined(SKNX_NO_SIMD)
         #include "../opts/Sk2x_sse.h"
-    #elif defined(__ARM_NEON__)                   && !defined(SKNX_NO_SIMD)
+    #elif defined(SK_ARM_HAS_NEON)                && !defined(SKNX_NO_SIMD)
         #include "../opts/Sk2x_neon.h"
     #else
         #include "../opts/Sk2x_none.h"
@@ -81,7 +81,7 @@ private:
 
 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE2 && !defined(SKNX_NO_SIMD)
     #include "../opts/Sk2x_sse.h"
-#elif defined(__ARM_NEON__)                   && !defined(SKNX_NO_SIMD)
+#elif defined(SK_ARM_HAS_NEON)                && !defined(SKNX_NO_SIMD)
     #include "../opts/Sk2x_neon.h"
 #else
     #include "../opts/Sk2x_none.h"
diff --git a/src/core/SkPMFloat.h b/src/core/SkPMFloat.h
index 04323ad1fc..699f85b2c4 100644
--- a/src/core/SkPMFloat.h
+++ b/src/core/SkPMFloat.h
@@ -8,7 +8,7 @@
 
 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE2
     #include <immintrin.h>
-#elif defined(__ARM_NEON__)
+#elif defined(SK_ARM_HAS_NEON)
     #include <arm_neon.h>
 #endif
 
@@ -66,7 +66,7 @@ private:
         float fColor[4];
 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE2
         __m128 fColors;
-#elif defined(__ARM_NEON__)
+#elif defined(SK_ARM_HAS_NEON)
         float32x4_t fColors;
 #endif
     };
@@ -76,7 +76,7 @@ private:
     #include "../opts/SkPMFloat_SSSE3.h"
 #elif SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE2
     #include "../opts/SkPMFloat_SSE2.h"
-#elif defined(__ARM_NEON__)
+#elif defined(SK_ARM_HAS_NEON)
     #include "../opts/SkPMFloat_neon.h"
 #else
     #include "../opts/SkPMFloat_none.h"
diff --git a/src/core/SkUtilsArm.h b/src/core/SkUtilsArm.h
index f15648136c..51ae7e4a95 100644
--- a/src/core/SkUtilsArm.h
+++ b/src/core/SkUtilsArm.h
@@ -21,9 +21,9 @@
 #define SK_ARM_NEON_MODE_ALWAYS   1
 #define SK_ARM_NEON_MODE_DYNAMIC  2
 
-#if defined(SK_CPU_ARM32) && defined(SK_ARM_HAS_OPTIONAL_NEON)
+#if defined(SK_ARM_HAS_OPTIONAL_NEON)
 #  define SK_ARM_NEON_MODE  SK_ARM_NEON_MODE_DYNAMIC
-#elif defined(SK_CPU_ARM32) && defined(SK_ARM_HAS_NEON) || defined(SK_CPU_ARM64)
+#elif defined(SK_ARM_HAS_NEON)
 #  define SK_ARM_NEON_MODE  SK_ARM_NEON_MODE_ALWAYS
 #else
 #  define SK_ARM_NEON_MODE  SK_ARM_NEON_MODE_NONE
diff --git a/src/opts/Sk2x_neon.h b/src/opts/Sk2x_neon.h
index cc4e799490..00ab00aeaa 100644
--- a/src/opts/Sk2x_neon.h
+++ b/src/opts/Sk2x_neon.h
@@ -15,7 +15,11 @@
     #include <math.h>
     template <typename T> struct SkScalarToSIMD;
     template <> struct SkScalarToSIMD< float> { typedef float32x2_t Type; };
-    template <> struct SkScalarToSIMD<double> { typedef double Type[2];   };
+    #if defined(SK_CPU_ARM64)
+        template <> struct SkScalarToSIMD<double> { typedef float64x2_t Type; };
+    #else
+        template <> struct SkScalarToSIMD<double> { typedef double Type[2];   };
+    #endif
 
 
 #elif defined(SK2X_PRIVATE)
@@ -28,10 +32,7 @@
 
 M() Sk2x() {}
 M() Sk2x(float val)        { fVec = vdup_n_f32(val);    }
-M() Sk2x(float a, float b) {
-    fVec = vset_lane_f32(a, fVec, 0);
-    fVec = vset_lane_f32(b, fVec, 1);
-}
+M() Sk2x(float a, float b) { fVec = (float32x2_t) { a, b }; }
 M(Sk2f&) operator=(const Sk2f& o) { fVec = o.fVec; return *this; }
 
 M(Sk2f) Load(const float vals[2]) { return vld1_f32(vals); }
@@ -60,33 +61,62 @@ M(Sk2f)  sqrt() const {
 
 #define M(...) template <> inline __VA_ARGS__ Sk2x<double>::
 
-// TODO: #ifdef SK_CPU_ARM64 use float64x2_t for Sk2d.
-
-M() Sk2x() {}
-M() Sk2x(double val)         { fVec[0] = fVec[1] = val; }
-M() Sk2x(double a, double b) { fVec[0] = a; fVec[1] = b; }
-M(Sk2d&) operator=(const Sk2d& o) {
-    fVec[0] = o.fVec[0];
-    fVec[1] = o.fVec[1];
-    return *this;
-}
-
-M(Sk2d) Load(const double vals[2]) { return Sk2d(vals[0], vals[1]); }
-M(void) store(double vals[2]) const { vals[0] = fVec[0]; vals[1] = fVec[1]; }
-
-M(Sk2d)      add(const Sk2d& o) const { return Sk2d(fVec[0] + o.fVec[0], fVec[1] + o.fVec[1]); }
-M(Sk2d) subtract(const Sk2d& o) const { return Sk2d(fVec[0] - o.fVec[0], fVec[1] - o.fVec[1]); }
-M(Sk2d) multiply(const Sk2d& o) const { return Sk2d(fVec[0] * o.fVec[0], fVec[1] * o.fVec[1]); }
-
-M(Sk2d) Min(const Sk2d& a, const Sk2d& b) {
-    return Sk2d(SkTMin(a.fVec[0], b.fVec[0]), SkTMin(a.fVec[1], b.fVec[1]));
-}
-M(Sk2d) Max(const Sk2d& a, const Sk2d& b) {
-    return Sk2d(SkTMax(a.fVec[0], b.fVec[0]), SkTMax(a.fVec[1], b.fVec[1]));
-}
-
-M(Sk2d) rsqrt() const { return Sk2d(1.0/::sqrt(fVec[0]), 1.0/::sqrt(fVec[1])); }
-M(Sk2d)  sqrt() const { return Sk2d(    ::sqrt(fVec[0]),     ::sqrt(fVec[1])); }
+#if defined(SK_CPU_ARM64)
+    M() Sk2x() {}
+    M() Sk2x(double val)         { fVec = vdupq_n_f64(val);    }
+    M() Sk2x(double a, double b) { fVec = (float64x2_t) { a, b }; }
+    M(Sk2d&) operator=(const Sk2d& o) { fVec = o.fVec; return *this; }
+
+    M(Sk2d) Load(const double vals[2]) { return vld1q_f64(vals); }
+    M(void) store(double vals[2]) const { vst1q_f64(vals, fVec); }
+
+    M(Sk2d)      add(const Sk2d& o) const { return vaddq_f64(fVec, o.fVec); }
+    M(Sk2d) subtract(const Sk2d& o) const { return vsubq_f64(fVec, o.fVec); }
+    M(Sk2d) multiply(const Sk2d& o) const { return vmulq_f64(fVec, o.fVec); }
+
+    M(Sk2d) Min(const Sk2d& a, const Sk2d& b) { return vminq_f64(a.fVec, b.fVec); }
+    M(Sk2d) Max(const Sk2d& a, const Sk2d& b) { return vmaxq_f64(a.fVec, b.fVec); }
+
+    M(Sk2d) rsqrt() const {
+        float64x2_t est0 = vrsqrteq_f64(fVec),
+                    est1 = vmulq_f64(vrsqrtsq_f64(fVec, vmulq_f64(est0, est0)), est0);
+        return est1;
+    }
+    M(Sk2d)  sqrt() const {
+        float64x2_t est1 = this->rsqrt().fVec,
+        // Two extra steps of Newton's method to refine the estimate of 1/sqrt(this).
+                    est2 = vmulq_f64(vrsqrtsq_f64(fVec, vmulq_f64(est1, est1)), est1),
+                    est3 = vmulq_f64(vrsqrtsq_f64(fVec, vmulq_f64(est2, est2)), est2);
+        return vmulq_f64(fVec, est3);
+    }
+
+#else  // Scalar implementation for 32-bit chips, which don't have float64x2_t.
+    M() Sk2x() {}
+    M() Sk2x(double val)         { fVec[0] = fVec[1] = val; }
+    M() Sk2x(double a, double b) { fVec[0] = a; fVec[1] = b; }
+    M(Sk2d&) operator=(const Sk2d& o) {
+        fVec[0] = o.fVec[0];
+        fVec[1] = o.fVec[1];
+        return *this;
+    }
+
+    M(Sk2d) Load(const double vals[2]) { return Sk2d(vals[0], vals[1]); }
+    M(void) store(double vals[2]) const { vals[0] = fVec[0]; vals[1] = fVec[1]; }
+
+    M(Sk2d)      add(const Sk2d& o) const { return Sk2d(fVec[0] + o.fVec[0], fVec[1] + o.fVec[1]); }
+    M(Sk2d) subtract(const Sk2d& o) const { return Sk2d(fVec[0] - o.fVec[0], fVec[1] - o.fVec[1]); }
+    M(Sk2d) multiply(const Sk2d& o) const { return Sk2d(fVec[0] * o.fVec[0], fVec[1] * o.fVec[1]); }
+
+    M(Sk2d) Min(const Sk2d& a, const Sk2d& b) {
+        return Sk2d(SkTMin(a.fVec[0], b.fVec[0]), SkTMin(a.fVec[1], b.fVec[1]));
+    }
+    M(Sk2d) Max(const Sk2d& a, const Sk2d& b) {
+        return Sk2d(SkTMax(a.fVec[0], b.fVec[0]), SkTMax(a.fVec[1], b.fVec[1]));
+    }
+
+    M(Sk2d) rsqrt() const { return Sk2d(1.0/::sqrt(fVec[0]), 1.0/::sqrt(fVec[1])); }
+    M(Sk2d)  sqrt() const { return Sk2d(    ::sqrt(fVec[0]),     ::sqrt(fVec[1])); }
+#endif
 
 #undef M
 
diff --git a/src/opts/Sk4x_neon.h b/src/opts/Sk4x_neon.h
index 3f35fe785b..92cde11711 100644
--- a/src/opts/Sk4x_neon.h
+++ b/src/opts/Sk4x_neon.h
@@ -37,20 +37,7 @@ template <typename T> Sk4x<T>& Sk4x<T>::operator=(const Sk4x<T>& other) {
 #define M(...) template <> inline __VA_ARGS__ Sk4f::
 
 M() Sk4x(float v) : fVec(vdupq_n_f32(v)) {}
-M() Sk4x(float a, float b, float c, float d) {
-    // NEON lacks an intrinsic to make this easy.  It is recommended to avoid
-    // this constructor unless it is absolutely necessary.
-
-    // I am choosing to use the set lane intrinsics.  Particularly, in the case
-    // of floating point, it is likely that the values are already in the right
-    // register file, so this may be the best approach.  However, I am not
-    // certain that this is the fastest approach and experimentation might be
-    // useful.
-    fVec = vsetq_lane_f32(a, fVec, 0);
-    fVec = vsetq_lane_f32(b, fVec, 1);
-    fVec = vsetq_lane_f32(c, fVec, 2);
-    fVec = vsetq_lane_f32(d, fVec, 3);
-}
+M() Sk4x(float a, float b, float c, float d) { fVec = (float32x4_t) { a, b, c, d }; }
 
 // As far as I can tell, it's not possible to provide an alignment hint to
 // NEON using intrinsics.  However, I think it is possible at the assembly
@@ -130,28 +117,7 @@ M(Sk4f) ZWCD(const Sk4f& xyzw, const Sk4f& abcd) {
 #define M(...) template <> inline __VA_ARGS__ Sk4i::
 
 M() Sk4x(int32_t v) : fVec(vdupq_n_s32(v)) {}
-M() Sk4x(int32_t a, int32_t b, int32_t c, int32_t d) {
-    // NEON lacks an intrinsic to make this easy.  It is recommended to avoid
-    // this constructor unless it is absolutely necessary.
-
-    // There are a few different implementation strategies.
-
-    // uint64_t ab_i = ((uint32_t) a) | (((uint64_t) b) << 32);
-    // uint64_t cd_i = ((uint32_t) c) | (((uint64_t) d) << 32);
-    // int32x2_t ab = vcreate_s32(ab_i);
-    // int32x2_t cd = vcreate_s32(cd_i);
-    // fVec = vcombine_s32(ab, cd);
-    // This might not be a bad idea for the integer case.  Either way I think,
-    // we will need to move values from general registers to NEON registers.
-
-    // I am choosing to use the set lane intrinsics.  I am not certain that
-    // this is the fastest approach.  It may be useful to try the above code
-    // for integers.
-    fVec = vsetq_lane_s32(a, fVec, 0);
-    fVec = vsetq_lane_s32(b, fVec, 1);
-    fVec = vsetq_lane_s32(c, fVec, 2);
-    fVec = vsetq_lane_s32(d, fVec, 3);
-}
+M() Sk4x(int32_t a, int32_t b, int32_t c, int32_t d) { fVec = (int32x4_t) { a, b, c, d }; }
 
 // As far as I can tell, it's not possible to provide an alignment hint to
 // NEON using intrinsics.  However, I think it is possible at the assembly