SkRasterPipeline: 8x pipelines, attempt 2

Original review here: https://skia-review.googlesource.com/c/2990/

Changes since:
  - simpler implementations of load_tail() / store_tail(): slower, but more obviously correct to all compilers
  - fleshed out math ops on Sk8i and Sk8u to make unit tests happy on -Fast bot (where we always have AVX2)
  - now storing stage functions as void(*)() to avoid undefined behavior and/or linker problems.  This restores 32-bit Windows.
  - all AVX2 Sk8x methods are marked always-inline, to avoid linking the "wrong" version on Debug builds.

CQ_INCLUDE_TRYBOTS=master.client.skia:Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-ASAN-Trybot,Perf-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Debug-GN,Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-SKNX_NO_SIMD-Trybot,Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Fast-Trybot;master.client.skia.compile:Build-Win-MSVC-x86_64-Debug-Trybot

GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=3064

Change-Id: Id0ba250037e271a9475fe2f0989d64f0aa909bae
Reviewed-on: https://skia-review.googlesource.com/3064
Reviewed-by: Mike Klein <mtklein@chromium.org>
Commit-Queue: Mike Klein <mtklein@chromium.org>

4 files changed, 520 insertions, 227 deletions

diff --git a/src/opts/SkNx_sse.h b/src/opts/SkNx_sse.h
index 66b5f0e9b3..4546280f02 100644
--- a/src/opts/SkNx_sse.h
+++ b/src/opts/SkNx_sse.h
@@ -302,12 +302,47 @@ public:
 
     SkNx() {}
     SkNx(uint16_t val) : fVec(_mm_set1_epi16(val)) {}
-    static SkNx Load(const void* ptr) { return _mm_loadu_si128((const __m128i*)ptr); }
     SkNx(uint16_t a, uint16_t b, uint16_t c, uint16_t d,
          uint16_t e, uint16_t f, uint16_t g, uint16_t h) : fVec(_mm_setr_epi16(a,b,c,d,e,f,g,h)) {}
 
+    static SkNx Load(const void* ptr) { return _mm_loadu_si128((const __m128i*)ptr); }
     void store(void* ptr) const { _mm_storeu_si128((__m128i*)ptr, fVec); }
 
+    static void Load4(const void* ptr, SkNx* r, SkNx* g, SkNx* b, SkNx* a) {
+        // TODO: AVX2 version
+        __m128i _01 = _mm_loadu_si128(((__m128i*)ptr) + 0),
+                _23 = _mm_loadu_si128(((__m128i*)ptr) + 1),
+                _45 = _mm_loadu_si128(((__m128i*)ptr) + 2),
+                _67 = _mm_loadu_si128(((__m128i*)ptr) + 3);
+
+        __m128i _02 = _mm_unpacklo_epi16(_01, _23),  // r0 r2 g0 g2 b0 b2 a0 a2
+                _13 = _mm_unpackhi_epi16(_01, _23),  // r1 r3 g1 g3 b1 b3 a1 a3
+                _46 = _mm_unpacklo_epi16(_45, _67),
+                _57 = _mm_unpackhi_epi16(_45, _67);
+
+        __m128i rg0123 = _mm_unpacklo_epi16(_02, _13),  // r0 r1 r2 r3 g0 g1 g2 g3
+                ba0123 = _mm_unpackhi_epi16(_02, _13),  // b0 b1 b2 b3 a0 a1 a2 a3
+                rg4567 = _mm_unpacklo_epi16(_46, _57),
+                ba4567 = _mm_unpackhi_epi16(_46, _57);
+
+        *r = _mm_unpacklo_epi64(rg0123, rg4567);
+        *g = _mm_unpackhi_epi64(rg0123, rg4567);
+        *b = _mm_unpacklo_epi64(ba0123, ba4567);
+        *a = _mm_unpackhi_epi64(ba0123, ba4567);
+    }
+    static void Store4(void* ptr, const SkNx& r, const SkNx& g, const SkNx& b, const SkNx& a) {
+        // TODO: AVX2 version
+        __m128i rg0123 = _mm_unpacklo_epi16(r.fVec, g.fVec),  // r0 g0 r1 g1 r2 g2 r3 g3
+                rg4567 = _mm_unpackhi_epi16(r.fVec, g.fVec),  // r4 g4 r5 g5 r6 g6 r7 g7
+                ba0123 = _mm_unpacklo_epi16(b.fVec, a.fVec),
+                ba4567 = _mm_unpackhi_epi16(b.fVec, a.fVec);
+
+        _mm_storeu_si128((__m128i*)ptr + 0, _mm_unpacklo_epi32(rg0123, ba0123));
+        _mm_storeu_si128((__m128i*)ptr + 1, _mm_unpackhi_epi32(rg0123, ba0123));
+        _mm_storeu_si128((__m128i*)ptr + 2, _mm_unpacklo_epi32(rg4567, ba4567));
+        _mm_storeu_si128((__m128i*)ptr + 3, _mm_unpackhi_epi32(rg4567, ba4567));
+    }
+
     SkNx operator + (const SkNx& o) const { return _mm_add_epi16(fVec, o.fVec); }
     SkNx operator - (const SkNx& o) const { return _mm_sub_epi16(fVec, o.fVec); }
     SkNx operator * (const SkNx& o) const { return _mm_mullo_epi16(fVec, o.fVec); }
@@ -403,6 +438,192 @@ public:
     __m128i fVec;
 };
 
+#if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_AVX2
+
+// There are two different SkNx<8, uint8_t>, SkNx<8, int32_t>, SkNx<8, uint32_t>, SkNx<8, float>:
+//   - the default paired SkNx<4, ...> versions used without AVX2
+//   - the native AVX2 versions.
+// It is important that we don't call methods for one from the other.
+// Usually these methods inline, but they don't always in Debug builds.
+// For now, try to fix this by marking all the AVX2 versions as always-inline.
+// We may want or need to extend this strategy to all SkNx methods.
+#define I SK_ALWAYS_INLINE
+
+    template <>
+    class SkNx<8, uint8_t> {
+    public:
+        I SkNx(const __m128i& vec) : fVec(vec) {}
+
+        I SkNx() {}
+        I SkNx(uint8_t v) : fVec(_mm_set1_epi8(v)) {}
+        I SkNx(uint8_t a, uint8_t b, uint8_t c, uint8_t d,
+               uint8_t e, uint8_t f, uint8_t g, uint8_t h)
+            : fVec(_mm_setr_epi8(a,b,c,d, e,f,g,h, 0,0,0,0, 0,0,0,0)) {}
+
+
+        I static SkNx Load(const void* ptr) { return _mm_loadl_epi64((const __m128i*)ptr); }
+        I void store(void* ptr) const { _mm_storel_epi64((__m128i*)ptr, fVec); }
+
+        I uint8_t operator[](int k) const {
+            SkASSERT(0 <= k && k < 8);
+            union { __m128i v; uint8_t us[16]; } pun = {fVec};
+            return pun.us[k&7];
+        }
+
+        __m128i fVec;
+    };
+
+    template <>
+    class SkNx<8, int32_t> {
+    public:
+        I SkNx(const __m256i& vec) : fVec(vec) {}
+
+        I SkNx() {}
+        I SkNx(int32_t v) : fVec(_mm256_set1_epi32(v)) {}
+        I SkNx(int32_t a, int32_t b, int32_t c, int32_t d,
+               int32_t e, int32_t f, int32_t g, int32_t h)
+            : fVec(_mm256_setr_epi32(a,b,c,d, e,f,g,h)) {}
+
+        I static SkNx Load(const void* ptr) { return _mm256_loadu_si256((const __m256i*)ptr); }
+        I void store(void* ptr) const { _mm256_storeu_si256((__m256i*)ptr, fVec); }
+
+        I SkNx operator + (const SkNx& o) const { return _mm256_add_epi32(fVec, o.fVec); }
+        I SkNx operator - (const SkNx& o) const { return _mm256_sub_epi32(fVec, o.fVec); }
+        I SkNx operator * (const SkNx& o) const { return _mm256_mullo_epi32(fVec, o.fVec); }
+
+        I SkNx operator & (const SkNx& o) const { return _mm256_and_si256(fVec, o.fVec); }
+        I SkNx operator | (const SkNx& o) const { return _mm256_or_si256(fVec, o.fVec); }
+        I SkNx operator ^ (const SkNx& o) const { return _mm256_xor_si256(fVec, o.fVec); }
+
+        I SkNx operator << (int bits) const { return _mm256_slli_epi32(fVec, bits); }
+        I SkNx operator >> (int bits) const { return _mm256_srai_epi32(fVec, bits); }
+
+        I int32_t operator[](int k) const {
+            SkASSERT(0 <= k && k < 8);
+            union { __m256i v; int32_t is[8]; } pun = {fVec};
+            return pun.is[k&7];
+        }
+
+        __m256i fVec;
+    };
+
+    template <>
+    class SkNx<8, uint32_t> {
+    public:
+        I SkNx(const __m256i& vec) : fVec(vec) {}
+
+        I SkNx() {}
+        I SkNx(uint32_t v) : fVec(_mm256_set1_epi32(v)) {}
+        I SkNx(uint32_t a, uint32_t b, uint32_t c, uint32_t d,
+               uint32_t e, uint32_t f, uint32_t g, uint32_t h)
+            : fVec(_mm256_setr_epi32(a,b,c,d, e,f,g,h)) {}
+
+        I static SkNx Load(const void* ptr) { return _mm256_loadu_si256((const __m256i*)ptr); }
+        I void store(void* ptr) const { _mm256_storeu_si256((__m256i*)ptr, fVec); }
+
+        I SkNx operator + (const SkNx& o) const { return _mm256_add_epi32(fVec, o.fVec); }
+        I SkNx operator - (const SkNx& o) const { return _mm256_sub_epi32(fVec, o.fVec); }
+        I SkNx operator * (const SkNx& o) const { return _mm256_mullo_epi32(fVec, o.fVec); }
+
+        I SkNx operator & (const SkNx& o) const { return _mm256_and_si256(fVec, o.fVec); }
+        I SkNx operator | (const SkNx& o) const { return _mm256_or_si256(fVec, o.fVec); }
+        I SkNx operator ^ (const SkNx& o) const { return _mm256_xor_si256(fVec, o.fVec); }
+
+        I SkNx operator << (int bits) const { return _mm256_slli_epi32(fVec, bits); }
+        I SkNx operator >> (int bits) const { return _mm256_srli_epi32(fVec, bits); }
+
+        I uint32_t operator[](int k) const {
+            SkASSERT(0 <= k && k < 8);
+            union { __m256i v; uint32_t us[8]; } pun = {fVec};
+            return pun.us[k&7];
+        }
+
+        __m256i fVec;
+    };
+
+    template <>
+    class SkNx<8, float> {
+    public:
+        I SkNx(const __m256& vec) : fVec(vec) {}
+
+        I SkNx() {}
+        I SkNx(float val) : fVec(_mm256_set1_ps(val)) {}
+        I SkNx(float a, float b, float c, float d,
+               float e, float f, float g, float h) : fVec(_mm256_setr_ps(a,b,c,d,e,f,g,h)) {}
+
+        I static SkNx Load(const void* ptr) { return _mm256_loadu_ps((const float*)ptr); }
+        I void store(void* ptr) const { _mm256_storeu_ps((float*)ptr, fVec); }
+
+        I SkNx operator+(const SkNx& o) const { return _mm256_add_ps(fVec, o.fVec); }
+        I SkNx operator-(const SkNx& o) const { return _mm256_sub_ps(fVec, o.fVec); }
+        I SkNx operator*(const SkNx& o) const { return _mm256_mul_ps(fVec, o.fVec); }
+        I SkNx operator/(const SkNx& o) const { return _mm256_div_ps(fVec, o.fVec); }
+
+        I SkNx operator==(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_EQ_OQ); }
+        I SkNx operator!=(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_NEQ_OQ); }
+        I SkNx operator <(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_LT_OQ); }
+        I SkNx operator >(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_GT_OQ); }
+        I SkNx operator<=(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_LE_OQ); }
+        I SkNx operator>=(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_GE_OQ); }
+
+        I static SkNx Min(const SkNx& l, const SkNx& r) { return _mm256_min_ps(l.fVec, r.fVec); }
+        I static SkNx Max(const SkNx& l, const SkNx& r) { return _mm256_max_ps(l.fVec, r.fVec); }
+
+        I SkNx   sqrt() const { return _mm256_sqrt_ps (fVec); }
+        I SkNx  rsqrt() const { return _mm256_rsqrt_ps(fVec); }
+        I SkNx invert() const { return _mm256_rcp_ps  (fVec); }
+
+        I float operator[](int k) const {
+            SkASSERT(0 <= k && k < 8);
+            union { __m256 v; float fs[8]; } pun = {fVec};
+            return pun.fs[k&7];
+        }
+
+        I SkNx thenElse(const SkNx& t, const SkNx& e) const {
+            return _mm256_blendv_ps(e.fVec, t.fVec, fVec);
+        }
+
+        __m256 fVec;
+    };
+
+    static I void SkNx_split(const Sk8f& v, Sk4f* lo, Sk4f* hi) {
+        *lo = _mm256_extractf128_ps(v.fVec, 0);
+        *hi = _mm256_extractf128_ps(v.fVec, 1);
+    }
+
+    static I Sk8f SkNx_join(const Sk4f& lo, const Sk4f& hi) {
+        return _mm256_insertf128_ps(_mm256_castps128_ps256(lo.fVec), hi.fVec, 1);
+    }
+
+    static I Sk8f SkNx_fma(const Sk8f& a, const Sk8f& b, const Sk8f& c) {
+        return _mm256_fmadd_ps(a.fVec, b.fVec, c.fVec);
+    }
+
+    template<> /*static*/ I Sk8f SkNx_cast<float>(const Sk8b& src) {
+        return _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(src.fVec));
+    }
+
+    template<> /*static*/ I Sk8f SkNx_cast<float>(const Sk8i& src) {
+        return _mm256_cvtepi32_ps(src.fVec);
+    }
+
+    template<> /*static*/ I Sk8i SkNx_cast<int>(const Sk8f& src) {
+        return _mm256_cvttps_epi32(src.fVec);
+    }
+
+    template<> /*static*/ I Sk8i SkNx_cast<int>(const Sk8h& src) {
+        return _mm256_cvtepu16_epi32(src.fVec);
+    }
+    template<> /*static*/ I Sk8h SkNx_cast<uint16_t>(const Sk8i& src) {
+        __m128i lo = _mm256_extractf128_si256(src.fVec, 0),
+                hi = _mm256_extractf128_si256(src.fVec, 1);
+        return _mm_packus_epi32(lo, hi);
+    }
+
+#undef I
+
+#endif
+
 template<> /*static*/ inline Sk4f SkNx_cast<float, int32_t>(const Sk4i& src) {
     return _mm_cvtepi32_ps(src.fVec);
 }
diff --git a/src/opts/SkOpts_hsw.cpp b/src/opts/SkOpts_hsw.cpp
index 53e2e5acdd..9f724805a6 100644
--- a/src/opts/SkOpts_hsw.cpp
+++ b/src/opts/SkOpts_hsw.cpp
@@ -7,9 +7,70 @@
 
 #include "SkOpts.h"
 
+
 #define SK_OPTS_NS hsw
+#include "SkRasterPipeline_opts.h"
 
 namespace SkOpts {
-    void Init_hsw() { }
+    void Init_hsw() {
+
+        run_pipeline = SK_OPTS_NS::run_pipeline;
+
+    #define STAGE(stage)                                                       \
+        body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage;      \
+        tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage##_tail
+
+        STAGE(store_565);
+        STAGE(store_srgb);
+        STAGE(store_f16);
+
+        STAGE(load_s_565);
+        STAGE(load_s_srgb);
+        STAGE(load_s_f16);
+
+        STAGE(load_d_565);
+        STAGE(load_d_srgb);
+        STAGE(load_d_f16);
+
+        STAGE(scale_u8);
+
+        STAGE(lerp_u8);
+        STAGE(lerp_565);
+    #undef STAGE
+
+    #define STAGE(stage) \
+        body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage; \
+        tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage
+
+        STAGE(lerp_constant_float);
+        STAGE(constant_color);
+
+        STAGE(dst);
+        STAGE(dstatop);
+        STAGE(dstin);
+        STAGE(dstout);
+        STAGE(dstover);
+        STAGE(srcatop);
+        STAGE(srcin);
+        STAGE(srcout);
+        STAGE(srcover);
+        STAGE(clear);
+        STAGE(modulate);
+        STAGE(multiply);
+        STAGE(plus_);
+        STAGE(screen);
+        STAGE(xor_);
+        STAGE(colorburn);
+        STAGE(colordodge);
+        STAGE(darken);
+        STAGE(difference);
+        STAGE(exclusion);
+        STAGE(hardlight);
+        STAGE(lighten);
+        STAGE(overlay);
+        STAGE(softlight);
+    #undef STAGE
+
+    }
 }
 
diff --git a/src/opts/SkOpts_sse41.cpp b/src/opts/SkOpts_sse41.cpp
index 3a37834c72..d55978c8b7 100644
--- a/src/opts/SkOpts_sse41.cpp
+++ b/src/opts/SkOpts_sse41.cpp
@@ -21,58 +21,60 @@ namespace SkOpts {
         srcover_srgb_srgb    = sse41::srcover_srgb_srgb;
         blit_row_s32a_opaque = sse41::blit_row_s32a_opaque;
 
-    #define STAGE(stage, kCallNext) \
-        stages_4  [SkRasterPipeline::stage] = stage_4  <SK_OPTS_NS::stage, kCallNext>; \
-        stages_1_3[SkRasterPipeline::stage] = stage_1_3<SK_OPTS_NS::stage, kCallNext>
+    #define STAGE(stage)                                                       \
+        body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage;      \
+        tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage##_tail
 
-        STAGE(store_565 , false);
-        STAGE(store_srgb, false);
-        STAGE(store_f16 , false);
+        STAGE(store_565);
+        STAGE(store_srgb);
+        STAGE(store_f16);
 
-        STAGE(load_s_565 , true);
-        STAGE(load_s_srgb, true);
-        STAGE(load_s_f16 , true);
+        STAGE(load_s_565);
+        STAGE(load_s_srgb);
+        STAGE(load_s_f16);
 
-        STAGE(load_d_565 , true);
-        STAGE(load_d_srgb, true);
-        STAGE(load_d_f16 , true);
+        STAGE(load_d_565);
+        STAGE(load_d_srgb);
+        STAGE(load_d_f16);
 
-        STAGE(scale_u8, true);
+        STAGE(scale_u8);
 
-        STAGE(lerp_u8            , true);
-        STAGE(lerp_565           , true);
-        STAGE(lerp_constant_float, true);
+        STAGE(lerp_u8);
+        STAGE(lerp_565);
+    #undef STAGE
 
-        STAGE(constant_color, true);
+    #define STAGE(stage)                                                  \
+        body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage; \
+        tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage
 
-    #undef STAGE
+      // The commented-out stages don't actually benefit from SSE 4.1.
+      // To cut down on code bloat we skip them here, using the identical SSE2 defaults.
 
-    #define STAGE(stage) \
-        stages_4  [SkRasterPipeline::stage] = SK_OPTS_NS::stage; \
-        stages_1_3[SkRasterPipeline::stage] = SK_OPTS_NS::stage
+      //STAGE(lerp_constant_float);
+      //STAGE(constant_color);
 
-        STAGE(dst);
-        STAGE(dstatop);
-        STAGE(dstin);
-        STAGE(dstout);
-        STAGE(dstover);
-        STAGE(srcatop);
-        STAGE(srcin);
-        STAGE(srcout);
-        STAGE(srcover);
-        STAGE(clear);
-        STAGE(modulate);
-        STAGE(multiply);
-        STAGE(plus_);
-        STAGE(screen);
-        STAGE(xor_);
+      //STAGE(dst);
+      //STAGE(dstatop);
+      //STAGE(dstin);
+      //STAGE(dstout);
+      //STAGE(dstover);
+      //STAGE(srcatop);
+      //STAGE(srcin);
+      //STAGE(srcout);
+      //STAGE(srcover);
+      //STAGE(clear);
+      //STAGE(modulate);
+      //STAGE(multiply);
+      //STAGE(plus_);
+      //STAGE(screen);
+      //STAGE(xor_);
         STAGE(colorburn);
         STAGE(colordodge);
-        STAGE(darken);
-        STAGE(difference);
-        STAGE(exclusion);
+      //STAGE(darken);
+      //STAGE(difference);
+      //STAGE(exclusion);
         STAGE(hardlight);
-        STAGE(lighten);
+      //STAGE(lighten);
         STAGE(overlay);
         STAGE(softlight);
     #undef STAGE
diff --git a/src/opts/SkRasterPipeline_opts.h b/src/opts/SkRasterPipeline_opts.h
index b0e6e1d2f9..1d8b04452d 100644
--- a/src/opts/SkRasterPipeline_opts.h
+++ b/src/opts/SkRasterPipeline_opts.h
@@ -13,129 +13,147 @@
 #include "SkRasterPipeline.h"
 #include "SkSRGB.h"
 
-using Kernel_Sk4f = void(void*, size_t, size_t, Sk4f&, Sk4f&, Sk4f&, Sk4f&,
-                                                Sk4f&, Sk4f&, Sk4f&, Sk4f&);
-
-// These are always static, and we _really_ want them to inline.
-// If you find yourself wanting a non-inline stage, write a SkRasterPipeline::Fn directly.
-#define KERNEL_Sk4f(name)                                                      \
-    static SK_ALWAYS_INLINE void name(void* ctx, size_t x, size_t tail,        \
-                                      Sk4f&  r, Sk4f&  g, Sk4f&  b, Sk4f&  a,  \
-                                      Sk4f& dr, Sk4f& dg, Sk4f& db, Sk4f& da)
-
-
-template <Kernel_Sk4f kernel, bool kCallNext>
-static inline void SK_VECTORCALL stage_4(SkRasterPipeline::Stage* st, size_t x, size_t tail,
-                                         Sk4f  r, Sk4f  g, Sk4f  b, Sk4f  a,
-                                         Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) {
-    // Passing 0 lets the optimizer completely drop any "if (tail) {...}" code in kernel.
-    kernel(st->ctx<void*>(), x,0, r,g,b,a, dr,dg,db,da);
-    if (kCallNext) {
-        st->next(x,tail, r,g,b,a, dr,dg,db,da);  // It's faster to pass t here than 0.
-    }
-}
+using SkNf = SkRasterPipeline::V;
+static constexpr auto N = sizeof(SkNf) / sizeof(float);
+using SkNi = SkNx<N, int>;
+using SkNh = SkNx<N, uint16_t>;
+
+#define SI static inline
+
+#define STAGE(name, kCallNext)                                                             \
+    static SK_ALWAYS_INLINE void name##_kernel(void* ctx, size_t x, size_t tail,           \
+                                               SkNf&  r, SkNf&  g, SkNf&  b, SkNf&  a,     \
+                                               SkNf& dr, SkNf& dg, SkNf& db, SkNf& da);    \
+    SI void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail,         \
+                               SkNf  r, SkNf  g, SkNf  b, SkNf  a,                         \
+                               SkNf dr, SkNf dg, SkNf db, SkNf da) {                       \
+        name##_kernel(st->ctx<void*>(), x,0, r,g,b,a, dr,dg,db,da);                        \
+        if (kCallNext) {                                                                   \
+            st->next(x,tail, r,g,b,a, dr,dg,db,da);                                        \
+        }                                                                                  \
+    }                                                                                      \
+    SI void SK_VECTORCALL name##_tail(SkRasterPipeline::Stage* st, size_t x, size_t tail,  \
+                                      SkNf  r, SkNf  g, SkNf  b, SkNf  a,                  \
+                                      SkNf dr, SkNf dg, SkNf db, SkNf da) {                \
+        name##_kernel(st->ctx<void*>(), x,tail, r,g,b,a, dr,dg,db,da);                     \
+        if (kCallNext) {                                                                   \
+            st->next(x,tail, r,g,b,a, dr,dg,db,da);                                        \
+        }                                                                                  \
+    }                                                                                      \
+    static SK_ALWAYS_INLINE void name##_kernel(void* ctx, size_t x, size_t tail,           \
+                                               SkNf&  r, SkNf&  g, SkNf&  b, SkNf&  a,     \
+                                               SkNf& dr, SkNf& dg, SkNf& db, SkNf& da)
 
-template <Kernel_Sk4f kernel, bool kCallNext>
-static inline void SK_VECTORCALL stage_1_3(SkRasterPipeline::Stage* st, size_t x, size_t tail,
-                                           Sk4f  r, Sk4f  g, Sk4f  b, Sk4f  a,
-                                           Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) {
-#if defined(__clang__)
-    __builtin_assume(tail > 0);  // This flourish lets Clang compile away any tail==0 code.
-#endif
-    kernel(st->ctx<void*>(), x,tail, r,g,b,a, dr,dg,db,da);
-    if (kCallNext) {
-        st->next(x,tail, r,g,b,a, dr,dg,db,da);
-    }
-}
 
 // Many xfermodes apply the same logic to each channel.
-#define RGBA_XFERMODE_Sk4f(name)                                                       \
-    static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa,          \
-                                               const Sk4f& d, const Sk4f& da);         \
-    static void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail, \
-                                   Sk4f  r, Sk4f  g, Sk4f  b, Sk4f  a,                 \
-                                   Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) {               \
-        r = name##_kernel(r,a,dr,da);                                                  \
-        g = name##_kernel(g,a,dg,da);                                                  \
-        b = name##_kernel(b,a,db,da);                                                  \
-        a = name##_kernel(a,a,da,da);                                                  \
-        st->next(x,tail, r,g,b,a, dr,dg,db,da);                                        \
-    }                                                                                  \
-    static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa,          \
-                                               const Sk4f& d, const Sk4f& da)
+#define RGBA_XFERMODE(name)                                                                \
+    static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa,              \
+                                               const SkNf& d, const SkNf& da);             \
+    SI void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail,         \
+                               SkNf  r, SkNf  g, SkNf  b, SkNf  a,                         \
+                               SkNf dr, SkNf dg, SkNf db, SkNf da) {                       \
+        r = name##_kernel(r,a,dr,da);                                                      \
+        g = name##_kernel(g,a,dg,da);                                                      \
+        b = name##_kernel(b,a,db,da);                                                      \
+        a = name##_kernel(a,a,da,da);                                                      \
+        st->next(x,tail, r,g,b,a, dr,dg,db,da);                                            \
+    }                                                                                      \
+    static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa,              \
+                                               const SkNf& d, const SkNf& da)
 
 // Most of the rest apply the same logic to color channels and use srcover's alpha logic.
-#define RGB_XFERMODE_Sk4f(name)                                                        \
-    static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa,          \
-                                               const Sk4f& d, const Sk4f& da);         \
-    static void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail, \
-                                   Sk4f  r, Sk4f  g, Sk4f  b, Sk4f  a,                 \
-                                   Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) {               \
-        r = name##_kernel(r,a,dr,da);                                                  \
-        g = name##_kernel(g,a,dg,da);                                                  \
-        b = name##_kernel(b,a,db,da);                                                  \
-        a = a + (da * (1.0f-a));                                                       \
-        st->next(x,tail, r,g,b,a, dr,dg,db,da);                                        \
-    }                                                                                  \
-    static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa,          \
-                                               const Sk4f& d, const Sk4f& da)
+#define RGB_XFERMODE(name)                                                                 \
+    static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa,              \
+                                               const SkNf& d, const SkNf& da);             \
+    SI void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail,         \
+                               SkNf  r, SkNf  g, SkNf  b, SkNf  a,                         \
+                               SkNf dr, SkNf dg, SkNf db, SkNf da) {                       \
+        r = name##_kernel(r,a,dr,da);                                                      \
+        g = name##_kernel(g,a,dg,da);                                                      \
+        b = name##_kernel(b,a,db,da);                                                      \
+        a = a + (da * (1.0f-a));                                                           \
+        st->next(x,tail, r,g,b,a, dr,dg,db,da);                                            \
+    }                                                                                      \
+    static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa,              \
+                                               const SkNf& d, const SkNf& da)
+
 
 namespace SK_OPTS_NS {
 
+    SI void run_pipeline(size_t x, size_t n,
+                         void (*vBodyStart)(), SkRasterPipeline::Stage* body,
+                         void (*vTailStart)(), SkRasterPipeline::Stage* tail) {
+        auto bodyStart = (SkRasterPipeline::Fn)vBodyStart,
+             tailStart = (SkRasterPipeline::Fn)vTailStart;
+        SkNf v;  // Fastest to start uninitialized.
+        while (n >= N) {
+            bodyStart(body, x,0, v,v,v,v, v,v,v,v);
+            x += N;
+            n -= N;
+        }
+        if (n > 0) {
+            tailStart(tail, x,n, v,v,v,v, v,v,v,v);
+        }
+    }
+
     // Clamp colors into [0,1] premul (e.g. just before storing back to memory).
-    static void clamp_01_premul(Sk4f& r, Sk4f& g, Sk4f& b, Sk4f& a) {
-        a = Sk4f::Max(a, 0.0f);
-        r = Sk4f::Max(r, 0.0f);
-        g = Sk4f::Max(g, 0.0f);
-        b = Sk4f::Max(b, 0.0f);
-
-        a = Sk4f::Min(a, 1.0f);
-        r = Sk4f::Min(r, a);
-        g = Sk4f::Min(g, a);
-        b = Sk4f::Min(b, a);
+    SI void clamp_01_premul(SkNf& r, SkNf& g, SkNf& b, SkNf& a) {
+        a = SkNf::Max(a, 0.0f);
+        r = SkNf::Max(r, 0.0f);
+        g = SkNf::Max(g, 0.0f);
+        b = SkNf::Max(b, 0.0f);
+
+        a = SkNf::Min(a, 1.0f);
+        r = SkNf::Min(r, a);
+        g = SkNf::Min(g, a);
+        b = SkNf::Min(b, a);
     }
 
-    static Sk4f inv(const Sk4f& x) { return 1.0f - x; }
+    SI SkNf inv(const SkNf& x) { return 1.0f - x; }
 
-    static Sk4f lerp(const Sk4f& from, const Sk4f& to, const Sk4f& cov) {
-        return from + (to-from)*cov;
+    SI SkNf lerp(const SkNf& from, const SkNf& to, const SkNf& cov) {
+        return SkNx_fma(to-from, cov, from);
     }
 
     template <typename T>
-    static SkNx<4,T> load_tail(size_t tail, const T* src) {
+    SI SkNx<N,T> load_tail(size_t tail, const T* src) {
+        // TODO: better tail, maskload for 32- and 64-bit T
+        T buf[N] = {0};
         if (tail) {
-           return SkNx<4,T>(src[0], (tail>1 ? src[1] : 0), (tail>2 ? src[2] : 0), 0);
+            memcpy(buf, src, tail*sizeof(T));
+            src = buf;
         }
-        return SkNx<4,T>::Load(src);
+        return SkNx<N,T>::Load(src);
     }
 
     template <typename T>
-    static void store_tail(size_t tail, const SkNx<4,T>& v, T* dst) {
-        switch(tail) {
-            case 0: return v.store(dst);
-            case 3: dst[2] = v[2];
-            case 2: dst[1] = v[1];
-            case 1: dst[0] = v[0];
+    SI void store_tail(size_t tail, const SkNx<N,T>& v, T* dst) {
+        // TODO: better tail, maskstore for 32- and 64-bit T
+        T buf[N] = {0};
+        v.store(tail ? buf : dst);
+        if (tail) {
+            memcpy(dst, buf, tail*sizeof(T));
         }
     }
 
-    static void from_565(const Sk4h& _565, Sk4f* r, Sk4f* g, Sk4f* b) {
-        Sk4i _32_bit = SkNx_cast<int>(_565);
+    SI void from_565(const SkNh& _565, SkNf* r, SkNf* g, SkNf* b) {
+        auto _32_bit = SkNx_cast<int>(_565);
 
         *r = SkNx_cast<float>(_32_bit & SK_R16_MASK_IN_PLACE) * (1.0f / SK_R16_MASK_IN_PLACE);
         *g = SkNx_cast<float>(_32_bit & SK_G16_MASK_IN_PLACE) * (1.0f / SK_G16_MASK_IN_PLACE);
         *b = SkNx_cast<float>(_32_bit & SK_B16_MASK_IN_PLACE) * (1.0f / SK_B16_MASK_IN_PLACE);
     }
 
-    static Sk4h to_565(const Sk4f& r, const Sk4f& g, const Sk4f& b) {
-        return SkNx_cast<uint16_t>( Sk4f_round(r * SK_R16_MASK) << SK_R16_SHIFT
-                                  | Sk4f_round(g * SK_G16_MASK) << SK_G16_SHIFT
-                                  | Sk4f_round(b * SK_B16_MASK) << SK_B16_SHIFT);
+    SI SkNh to_565(const SkNf& r, const SkNf& g, const SkNf& b) {
+        return SkNx_cast<uint16_t>( SkNx_cast<int>(r * SK_R16_MASK + 0.5f) << SK_R16_SHIFT
+                                  | SkNx_cast<int>(g * SK_G16_MASK + 0.5f) << SK_G16_SHIFT
+                                  | SkNx_cast<int>(b * SK_B16_MASK + 0.5f) << SK_B16_SHIFT);
     }
 
+    STAGE(just_return, false) { }
 
     // The default shader produces a constant color (from the SkPaint).
-    KERNEL_Sk4f(constant_color) {
+    STAGE(constant_color, true) {
         auto color = (const SkPM4f*)ctx;
         r = color->r();
         g = color->g();
@@ -144,8 +162,8 @@ namespace SK_OPTS_NS {
     }
 
     // s' = d(1-c) + sc, for a constant c.
-    KERNEL_Sk4f(lerp_constant_float) {
-        Sk4f c = *(const float*)ctx;
+    STAGE(lerp_constant_float, true) {
+        SkNf c = *(const float*)ctx;
 
         r = lerp(dr, r, c);
         g = lerp(dg, g, c);
@@ -154,10 +172,10 @@ namespace SK_OPTS_NS {
     }
 
     // s' = sc for 8-bit c.
-    KERNEL_Sk4f(scale_u8) {
+    STAGE(scale_u8, true) {
         auto ptr = (const uint8_t*)ctx + x;
 
-        Sk4f c = SkNx_cast<float>(load_tail(tail, ptr)) * (1/255.0f);
+        SkNf c = SkNx_cast<float>(load_tail(tail, ptr)) * (1/255.0f);
         r = r*c;
         g = g*c;
         b = b*c;
@@ -165,10 +183,10 @@ namespace SK_OPTS_NS {
     }
 
     // s' = d(1-c) + sc for 8-bit c.
-    KERNEL_Sk4f(lerp_u8) {
+    STAGE(lerp_u8, true) {
         auto ptr = (const uint8_t*)ctx + x;
 
-        Sk4f c = SkNx_cast<float>(load_tail(tail, ptr)) * (1/255.0f);
+        SkNf c = SkNx_cast<float>(load_tail(tail, ptr)) * (1/255.0f);
         r = lerp(dr, r, c);
         g = lerp(dg, g, c);
         b = lerp(db, b, c);
@@ -176,9 +194,9 @@ namespace SK_OPTS_NS {
     }
 
     // s' = d(1-c) + sc for 565 c.
-    KERNEL_Sk4f(lerp_565) {
+    STAGE(lerp_565, true) {
         auto ptr = (const uint16_t*)ctx + x;
-        Sk4f cr, cg, cb;
+        SkNf cr, cg, cb;
         from_565(load_tail(tail, ptr), &cr, &cg, &cb);
 
         r = lerp(dr, r, cr);
@@ -187,155 +205,145 @@ namespace SK_OPTS_NS {
         a = 1.0f;
     }
 
-    KERNEL_Sk4f(load_d_565) {
+    STAGE(load_d_565, true) {
         auto ptr = (const uint16_t*)ctx + x;
         from_565(load_tail(tail, ptr), &dr,&dg,&db);
         da = 1.0f;
     }
 
-    KERNEL_Sk4f(load_s_565) {
+    STAGE(load_s_565, true) {
         auto ptr = (const uint16_t*)ctx + x;
         from_565(load_tail(tail, ptr), &r,&g,&b);
         a = 1.0f;
     }
 
-    KERNEL_Sk4f(store_565) {
+    STAGE(store_565, false) {
         clamp_01_premul(r,g,b,a);
         auto ptr = (uint16_t*)ctx + x;
         store_tail(tail, to_565(r,g,b), ptr);
     }
 
-    KERNEL_Sk4f(load_d_f16) {
+    STAGE(load_d_f16, true) {
         auto ptr = (const uint64_t*)ctx + x;
 
+        uint64_t buf[N] = {0};
         if (tail) {
-            auto p0 =          SkHalfToFloat_finite_ftz(ptr[0])          ,
-                 p1 = tail>1 ? SkHalfToFloat_finite_ftz(ptr[1]) : Sk4f{0},
-                 p2 = tail>2 ? SkHalfToFloat_finite_ftz(ptr[2]) : Sk4f{0};
-            dr = { p0[0],p1[0],p2[0],0 };
-            dg = { p0[1],p1[1],p2[1],0 };
-            db = { p0[2],p1[2],p2[2],0 };
-            da = { p0[3],p1[3],p2[3],0 };
-            return;
+            memcpy(buf, ptr, tail*sizeof(uint64_t));
+            ptr = buf;
         }
 
-        Sk4h rh, gh, bh, ah;
-        Sk4h::Load4(ptr, &rh, &gh, &bh, &ah);
+        SkNh rh, gh, bh, ah;
+        SkNh::Load4(ptr, &rh, &gh, &bh, &ah);
         dr = SkHalfToFloat_finite_ftz(rh);
         dg = SkHalfToFloat_finite_ftz(gh);
         db = SkHalfToFloat_finite_ftz(bh);
         da = SkHalfToFloat_finite_ftz(ah);
     }
 
-    KERNEL_Sk4f(load_s_f16) {
+    STAGE(load_s_f16, true) {
         auto ptr = (const uint64_t*)ctx + x;
 
+        uint64_t buf[N] = {0};
         if (tail) {
-            auto p0 =          SkHalfToFloat_finite_ftz(ptr[0])          ,
-                 p1 = tail>1 ? SkHalfToFloat_finite_ftz(ptr[1]) : Sk4f{0},
-                 p2 = tail>2 ? SkHalfToFloat_finite_ftz(ptr[2]) : Sk4f{0};
-            r = { p0[0],p1[0],p2[0],0 };
-            g = { p0[1],p1[1],p2[1],0 };
-            b = { p0[2],p1[2],p2[2],0 };
-            a = { p0[3],p1[3],p2[3],0 };
-            return;
+            memcpy(buf, ptr, tail*sizeof(uint64_t));
+            ptr = buf;
         }
 
-        Sk4h rh, gh, bh, ah;
-        Sk4h::Load4(ptr, &rh, &gh, &bh, &ah);
+        SkNh rh, gh, bh, ah;
+        SkNh::Load4(ptr, &rh, &gh, &bh, &ah);
         r = SkHalfToFloat_finite_ftz(rh);
         g = SkHalfToFloat_finite_ftz(gh);
         b = SkHalfToFloat_finite_ftz(bh);
         a = SkHalfToFloat_finite_ftz(ah);
     }
 
-    KERNEL_Sk4f(store_f16) {
+    STAGE(store_f16, false) {
         clamp_01_premul(r,g,b,a);
         auto ptr = (uint64_t*)ctx + x;
 
-        switch (tail) {
-            case 0: return Sk4h::Store4(ptr, SkFloatToHalf_finite_ftz(r),
-                                             SkFloatToHalf_finite_ftz(g),
-                                             SkFloatToHalf_finite_ftz(b),
-                                             SkFloatToHalf_finite_ftz(a));
-
-            case 3: SkFloatToHalf_finite_ftz({r[2], g[2], b[2], a[2]}).store(ptr+2);
-            case 2: SkFloatToHalf_finite_ftz({r[1], g[1], b[1], a[1]}).store(ptr+1);
-            case 1: SkFloatToHalf_finite_ftz({r[0], g[0], b[0], a[0]}).store(ptr+0);
+        uint64_t buf[N] = {0};
+        SkNh::Store4(tail ? buf : ptr, SkFloatToHalf_finite_ftz(r),
+                                       SkFloatToHalf_finite_ftz(g),
+                                       SkFloatToHalf_finite_ftz(b),
+                                       SkFloatToHalf_finite_ftz(a));
+        if (tail) {
+            memcpy(ptr, buf, tail*sizeof(uint64_t));
         }
     }
 
 
     // Load 8-bit SkPMColor-order sRGB.
-    KERNEL_Sk4f(load_d_srgb) {
+    STAGE(load_d_srgb, true) {
         auto ptr = (const uint32_t*)ctx + x;
 
-        auto px = load_tail(tail, (const int*)ptr);
-        dr =    sk_linear_from_srgb_math((px >> SK_R32_SHIFT) & 0xff);
-        dg =    sk_linear_from_srgb_math((px >> SK_G32_SHIFT) & 0xff);
-        db =    sk_linear_from_srgb_math((px >> SK_B32_SHIFT) & 0xff);
-        da = (1/255.0f)*SkNx_cast<float>((px >> SK_A32_SHIFT) & 0xff);
+        auto px = load_tail(tail, ptr);
+        auto to_int = [](const SkNx<N, uint32_t>& v) { return SkNi::Load(&v); };
+        dr =    sk_linear_from_srgb_math(to_int((px >> SK_R32_SHIFT) & 0xff));
+        dg =    sk_linear_from_srgb_math(to_int((px >> SK_G32_SHIFT) & 0xff));
+        db =    sk_linear_from_srgb_math(to_int((px >> SK_B32_SHIFT) & 0xff));
+        da = (1/255.0f)*SkNx_cast<float>(to_int( px >> SK_A32_SHIFT        ));
     }
 
-    KERNEL_Sk4f(load_s_srgb) {
+    STAGE(load_s_srgb, true) {
         auto ptr = (const uint32_t*)ctx + x;
 
-        auto px = load_tail(tail, (const int*)ptr);
-        r =    sk_linear_from_srgb_math((px >> SK_R32_SHIFT) & 0xff);
-        g =    sk_linear_from_srgb_math((px >> SK_G32_SHIFT) & 0xff);
-        b =    sk_linear_from_srgb_math((px >> SK_B32_SHIFT) & 0xff);
-        a = (1/255.0f)*SkNx_cast<float>((px >> SK_A32_SHIFT) & 0xff);
+        auto px = load_tail(tail, ptr);
+        auto to_int = [](const SkNx<N, uint32_t>& v) { return SkNi::Load(&v); };
+        r =    sk_linear_from_srgb_math(to_int((px >> SK_R32_SHIFT) & 0xff));
+        g =    sk_linear_from_srgb_math(to_int((px >> SK_G32_SHIFT) & 0xff));
+        b =    sk_linear_from_srgb_math(to_int((px >> SK_B32_SHIFT) & 0xff));
+        a = (1/255.0f)*SkNx_cast<float>(to_int( px >> SK_A32_SHIFT        ));
     }
 
-    KERNEL_Sk4f(store_srgb) {
+    STAGE(store_srgb, false) {
         clamp_01_premul(r,g,b,a);
         auto ptr = (uint32_t*)ctx + x;
-        store_tail(tail, ( sk_linear_to_srgb_noclamp(r) << SK_R32_SHIFT
-                         | sk_linear_to_srgb_noclamp(g) << SK_G32_SHIFT
-                         | sk_linear_to_srgb_noclamp(b) << SK_B32_SHIFT
-                         |       Sk4f_round(255.0f * a) << SK_A32_SHIFT), (int*)ptr);
+        store_tail(tail, (      sk_linear_to_srgb_noclamp(r) << SK_R32_SHIFT
+                         |      sk_linear_to_srgb_noclamp(g) << SK_G32_SHIFT
+                         |      sk_linear_to_srgb_noclamp(b) << SK_B32_SHIFT
+                         | SkNx_cast<int>(255.0f * a + 0.5f) << SK_A32_SHIFT ), (int*)ptr);
     }
 
-    RGBA_XFERMODE_Sk4f(clear)    { return 0.0f; }
-  //RGBA_XFERMODE_Sk4f(src)      { return s; }   // This would be a no-op stage, so we just omit it.
-    RGBA_XFERMODE_Sk4f(dst)      { return d; }
-
-    RGBA_XFERMODE_Sk4f(srcatop)  { return s*da + d*inv(sa); }
-    RGBA_XFERMODE_Sk4f(srcin)    { return s * da; }
-    RGBA_XFERMODE_Sk4f(srcout)   { return s * inv(da); }
-    RGBA_XFERMODE_Sk4f(srcover)  { return s + inv(sa)*d; }
-    RGBA_XFERMODE_Sk4f(dstatop)  { return srcatop_kernel(d,da,s,sa); }
-    RGBA_XFERMODE_Sk4f(dstin)    { return srcin_kernel  (d,da,s,sa); }
-    RGBA_XFERMODE_Sk4f(dstout)   { return srcout_kernel (d,da,s,sa); }
-    RGBA_XFERMODE_Sk4f(dstover)  { return srcover_kernel(d,da,s,sa); }
-
-    RGBA_XFERMODE_Sk4f(modulate) { return s*d; }
-    RGBA_XFERMODE_Sk4f(multiply) { return s*inv(da) + d*inv(sa) + s*d; }
-    RGBA_XFERMODE_Sk4f(plus_)    { return s + d; }
-    RGBA_XFERMODE_Sk4f(screen)   { return s + d - s*d; }
-    RGBA_XFERMODE_Sk4f(xor_)     { return s*inv(da) + d*inv(sa); }
-
-    RGB_XFERMODE_Sk4f(colorburn) {
+    RGBA_XFERMODE(clear)    { return 0.0f; }
+  //RGBA_XFERMODE(src)      { return s; }   // This would be a no-op stage, so we just omit it.
+    RGBA_XFERMODE(dst)      { return d; }
+
+    RGBA_XFERMODE(srcatop)  { return s*da + d*inv(sa); }
+    RGBA_XFERMODE(srcin)    { return s * da; }
+    RGBA_XFERMODE(srcout)   { return s * inv(da); }
+    RGBA_XFERMODE(srcover)  { return SkNx_fma(d, inv(sa), s); }
+    RGBA_XFERMODE(dstatop)  { return srcatop_kernel(d,da,s,sa); }
+    RGBA_XFERMODE(dstin)    { return srcin_kernel  (d,da,s,sa); }
+    RGBA_XFERMODE(dstout)   { return srcout_kernel (d,da,s,sa); }
+    RGBA_XFERMODE(dstover)  { return srcover_kernel(d,da,s,sa); }
+
+    RGBA_XFERMODE(modulate) { return s*d; }
+    RGBA_XFERMODE(multiply) { return s*inv(da) + d*inv(sa) + s*d; }
+    RGBA_XFERMODE(plus_)    { return s + d; }
+    RGBA_XFERMODE(screen)   { return s + d - s*d; }
+    RGBA_XFERMODE(xor_)     { return s*inv(da) + d*inv(sa); }
+
+    RGB_XFERMODE(colorburn) {
         return (d == da  ).thenElse(d + s*inv(da),
                (s == 0.0f).thenElse(s + d*inv(sa),
-                                    sa*(da - Sk4f::Min(da, (da-d)*sa/s)) + s*inv(da) + d*inv(sa)));
+                                    sa*(da - SkNf::Min(da, (da-d)*sa/s)) + s*inv(da) + d*inv(sa)));
     }
-    RGB_XFERMODE_Sk4f(colordodge) {
+    RGB_XFERMODE(colordodge) {
         return (d == 0.0f).thenElse(d + s*inv(da),
                (s == sa  ).thenElse(s + d*inv(sa),
-                                    sa*Sk4f::Min(da, (d*sa)/(sa - s)) + s*inv(da) + d*inv(sa)));
+                                    sa*SkNf::Min(da, (d*sa)/(sa - s)) + s*inv(da) + d*inv(sa)));
     }
-    RGB_XFERMODE_Sk4f(darken)     { return s + d - Sk4f::Max(s*da, d*sa); }
-    RGB_XFERMODE_Sk4f(difference) { return s + d - 2.0f*Sk4f::Min(s*da,d*sa); }
-    RGB_XFERMODE_Sk4f(exclusion)  { return s + d - 2.0f*s*d; }
-    RGB_XFERMODE_Sk4f(hardlight) {
+    RGB_XFERMODE(darken)     { return s + d - SkNf::Max(s*da, d*sa); }
+    RGB_XFERMODE(difference) { return s + d - 2.0f*SkNf::Min(s*da,d*sa); }
+    RGB_XFERMODE(exclusion)  { return s + d - 2.0f*s*d; }
+    RGB_XFERMODE(hardlight) {
         return s*inv(da) + d*inv(sa)
              + (2.0f*s <= sa).thenElse(2.0f*s*d, sa*da - 2.0f*(da-d)*(sa-s));
     }
-    RGB_XFERMODE_Sk4f(lighten) { return s + d - Sk4f::Min(s*da, d*sa); }
-    RGB_XFERMODE_Sk4f(overlay) { return hardlight_kernel(d,da,s,sa); }
-    RGB_XFERMODE_Sk4f(softlight) {
-        Sk4f m  = (da > 0.0f).thenElse(d / da, 0.0f),
+    RGB_XFERMODE(lighten) { return s + d - SkNf::Min(s*da, d*sa); }
+    RGB_XFERMODE(overlay) { return hardlight_kernel(d,da,s,sa); }
+    RGB_XFERMODE(softlight) {
+        SkNf m  = (da > 0.0f).thenElse(d / da, 0.0f),
              s2 = 2.0f*s,
              m4 = 4.0f*m;
 
@@ -343,7 +351,7 @@ namespace SK_OPTS_NS {
         //    1. dark src?
         //    2. light src, dark dst?
         //    3. light src, light dst?
-        Sk4f darkSrc = d*(sa + (s2 - sa)*(1.0f - m)),     // Used in case 1.
+        SkNf darkSrc = d*(sa + (s2 - sa)*(1.0f - m)),     // Used in case 1.
              darkDst = (m4*m4 + m4)*(m - 1.0f) + 7.0f*m,  // Used in case 2.
              liteDst = m.rsqrt().invert() - m,            // Used in case 3.
              liteSrc = d*sa + da*(s2 - sa) * (4.0f*d <= da).thenElse(darkDst, liteDst);  // 2 or 3?
@@ -351,8 +359,9 @@ namespace SK_OPTS_NS {
     }
 }
 
-#undef KERNEL_Sk4f
-#undef RGB_XFERMODE_Sk4f
-#undef RGB_XFERMODE_Sk4f
+#undef SI
+#undef STAGE
+#undef RGBA_XFERMODE
+#undef RGB_XFERMODE
 
 #endif//SkRasterPipeline_opts_DEFINED