Revert "SkRasterPipeline: 8x pipelines, attempt 2"

This reverts commit Id0ba250037e271a9475fe2f0989d64f0aa909bae.

crbug.com/654213 
Looks like Chrome Canary's picking up Haswell code on non-Haswell machines.

Change-Id: I16f976da24db86d5c99636c472ffad56db213a2a
Reviewed-on: https://skia-review.googlesource.com/3108
Commit-Queue: Mike Klein <mtklein@chromium.org>
Reviewed-by: Mike Klein <mtklein@chromium.org>

4 files changed, 227 insertions, 520 deletions

diff --git a/src/opts/SkNx_sse.h b/src/opts/SkNx_sse.h
index 4546280f02..66b5f0e9b3 100644
--- a/src/opts/SkNx_sse.h
+++ b/src/opts/SkNx_sse.h
@@ -302,47 +302,12 @@ public:
 
     SkNx() {}
     SkNx(uint16_t val) : fVec(_mm_set1_epi16(val)) {}
+    static SkNx Load(const void* ptr) { return _mm_loadu_si128((const __m128i*)ptr); }
     SkNx(uint16_t a, uint16_t b, uint16_t c, uint16_t d,
          uint16_t e, uint16_t f, uint16_t g, uint16_t h) : fVec(_mm_setr_epi16(a,b,c,d,e,f,g,h)) {}
 
-    static SkNx Load(const void* ptr) { return _mm_loadu_si128((const __m128i*)ptr); }
     void store(void* ptr) const { _mm_storeu_si128((__m128i*)ptr, fVec); }
 
-    static void Load4(const void* ptr, SkNx* r, SkNx* g, SkNx* b, SkNx* a) {
-        // TODO: AVX2 version
-        __m128i _01 = _mm_loadu_si128(((__m128i*)ptr) + 0),
-                _23 = _mm_loadu_si128(((__m128i*)ptr) + 1),
-                _45 = _mm_loadu_si128(((__m128i*)ptr) + 2),
-                _67 = _mm_loadu_si128(((__m128i*)ptr) + 3);
-
-        __m128i _02 = _mm_unpacklo_epi16(_01, _23),  // r0 r2 g0 g2 b0 b2 a0 a2
-                _13 = _mm_unpackhi_epi16(_01, _23),  // r1 r3 g1 g3 b1 b3 a1 a3
-                _46 = _mm_unpacklo_epi16(_45, _67),
-                _57 = _mm_unpackhi_epi16(_45, _67);
-
-        __m128i rg0123 = _mm_unpacklo_epi16(_02, _13),  // r0 r1 r2 r3 g0 g1 g2 g3
-                ba0123 = _mm_unpackhi_epi16(_02, _13),  // b0 b1 b2 b3 a0 a1 a2 a3
-                rg4567 = _mm_unpacklo_epi16(_46, _57),
-                ba4567 = _mm_unpackhi_epi16(_46, _57);
-
-        *r = _mm_unpacklo_epi64(rg0123, rg4567);
-        *g = _mm_unpackhi_epi64(rg0123, rg4567);
-        *b = _mm_unpacklo_epi64(ba0123, ba4567);
-        *a = _mm_unpackhi_epi64(ba0123, ba4567);
-    }
-    static void Store4(void* ptr, const SkNx& r, const SkNx& g, const SkNx& b, const SkNx& a) {
-        // TODO: AVX2 version
-        __m128i rg0123 = _mm_unpacklo_epi16(r.fVec, g.fVec),  // r0 g0 r1 g1 r2 g2 r3 g3
-                rg4567 = _mm_unpackhi_epi16(r.fVec, g.fVec),  // r4 g4 r5 g5 r6 g6 r7 g7
-                ba0123 = _mm_unpacklo_epi16(b.fVec, a.fVec),
-                ba4567 = _mm_unpackhi_epi16(b.fVec, a.fVec);
-
-        _mm_storeu_si128((__m128i*)ptr + 0, _mm_unpacklo_epi32(rg0123, ba0123));
-        _mm_storeu_si128((__m128i*)ptr + 1, _mm_unpackhi_epi32(rg0123, ba0123));
-        _mm_storeu_si128((__m128i*)ptr + 2, _mm_unpacklo_epi32(rg4567, ba4567));
-        _mm_storeu_si128((__m128i*)ptr + 3, _mm_unpackhi_epi32(rg4567, ba4567));
-    }
-
     SkNx operator + (const SkNx& o) const { return _mm_add_epi16(fVec, o.fVec); }
     SkNx operator - (const SkNx& o) const { return _mm_sub_epi16(fVec, o.fVec); }
     SkNx operator * (const SkNx& o) const { return _mm_mullo_epi16(fVec, o.fVec); }
@@ -438,192 +403,6 @@ public:
     __m128i fVec;
 };
 
-#if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_AVX2
-
-// There are two different SkNx<8, uint8_t>, SkNx<8, int32_t>, SkNx<8, uint32_t>, SkNx<8, float>:
-//   - the default paired SkNx<4, ...> versions used without AVX2
-//   - the native AVX2 versions.
-// It is important that we don't call methods for one from the other.
-// Usually these methods inline, but they don't always in Debug builds.
-// For now, try to fix this by marking all the AVX2 versions as always-inline.
-// We may want or need to extend this strategy to all SkNx methods.
-#define I SK_ALWAYS_INLINE
-
-    template <>
-    class SkNx<8, uint8_t> {
-    public:
-        I SkNx(const __m128i& vec) : fVec(vec) {}
-
-        I SkNx() {}
-        I SkNx(uint8_t v) : fVec(_mm_set1_epi8(v)) {}
-        I SkNx(uint8_t a, uint8_t b, uint8_t c, uint8_t d,
-               uint8_t e, uint8_t f, uint8_t g, uint8_t h)
-            : fVec(_mm_setr_epi8(a,b,c,d, e,f,g,h, 0,0,0,0, 0,0,0,0)) {}
-
-
-        I static SkNx Load(const void* ptr) { return _mm_loadl_epi64((const __m128i*)ptr); }
-        I void store(void* ptr) const { _mm_storel_epi64((__m128i*)ptr, fVec); }
-
-        I uint8_t operator[](int k) const {
-            SkASSERT(0 <= k && k < 8);
-            union { __m128i v; uint8_t us[16]; } pun = {fVec};
-            return pun.us[k&7];
-        }
-
-        __m128i fVec;
-    };
-
-    template <>
-    class SkNx<8, int32_t> {
-    public:
-        I SkNx(const __m256i& vec) : fVec(vec) {}
-
-        I SkNx() {}
-        I SkNx(int32_t v) : fVec(_mm256_set1_epi32(v)) {}
-        I SkNx(int32_t a, int32_t b, int32_t c, int32_t d,
-               int32_t e, int32_t f, int32_t g, int32_t h)
-            : fVec(_mm256_setr_epi32(a,b,c,d, e,f,g,h)) {}
-
-        I static SkNx Load(const void* ptr) { return _mm256_loadu_si256((const __m256i*)ptr); }
-        I void store(void* ptr) const { _mm256_storeu_si256((__m256i*)ptr, fVec); }
-
-        I SkNx operator + (const SkNx& o) const { return _mm256_add_epi32(fVec, o.fVec); }
-        I SkNx operator - (const SkNx& o) const { return _mm256_sub_epi32(fVec, o.fVec); }
-        I SkNx operator * (const SkNx& o) const { return _mm256_mullo_epi32(fVec, o.fVec); }
-
-        I SkNx operator & (const SkNx& o) const { return _mm256_and_si256(fVec, o.fVec); }
-        I SkNx operator | (const SkNx& o) const { return _mm256_or_si256(fVec, o.fVec); }
-        I SkNx operator ^ (const SkNx& o) const { return _mm256_xor_si256(fVec, o.fVec); }
-
-        I SkNx operator << (int bits) const { return _mm256_slli_epi32(fVec, bits); }
-        I SkNx operator >> (int bits) const { return _mm256_srai_epi32(fVec, bits); }
-
-        I int32_t operator[](int k) const {
-            SkASSERT(0 <= k && k < 8);
-            union { __m256i v; int32_t is[8]; } pun = {fVec};
-            return pun.is[k&7];
-        }
-
-        __m256i fVec;
-    };
-
-    template <>
-    class SkNx<8, uint32_t> {
-    public:
-        I SkNx(const __m256i& vec) : fVec(vec) {}
-
-        I SkNx() {}
-        I SkNx(uint32_t v) : fVec(_mm256_set1_epi32(v)) {}
-        I SkNx(uint32_t a, uint32_t b, uint32_t c, uint32_t d,
-               uint32_t e, uint32_t f, uint32_t g, uint32_t h)
-            : fVec(_mm256_setr_epi32(a,b,c,d, e,f,g,h)) {}
-
-        I static SkNx Load(const void* ptr) { return _mm256_loadu_si256((const __m256i*)ptr); }
-        I void store(void* ptr) const { _mm256_storeu_si256((__m256i*)ptr, fVec); }
-
-        I SkNx operator + (const SkNx& o) const { return _mm256_add_epi32(fVec, o.fVec); }
-        I SkNx operator - (const SkNx& o) const { return _mm256_sub_epi32(fVec, o.fVec); }
-        I SkNx operator * (const SkNx& o) const { return _mm256_mullo_epi32(fVec, o.fVec); }
-
-        I SkNx operator & (const SkNx& o) const { return _mm256_and_si256(fVec, o.fVec); }
-        I SkNx operator | (const SkNx& o) const { return _mm256_or_si256(fVec, o.fVec); }
-        I SkNx operator ^ (const SkNx& o) const { return _mm256_xor_si256(fVec, o.fVec); }
-
-        I SkNx operator << (int bits) const { return _mm256_slli_epi32(fVec, bits); }
-        I SkNx operator >> (int bits) const { return _mm256_srli_epi32(fVec, bits); }
-
-        I uint32_t operator[](int k) const {
-            SkASSERT(0 <= k && k < 8);
-            union { __m256i v; uint32_t us[8]; } pun = {fVec};
-            return pun.us[k&7];
-        }
-
-        __m256i fVec;
-    };
-
-    template <>
-    class SkNx<8, float> {
-    public:
-        I SkNx(const __m256& vec) : fVec(vec) {}
-
-        I SkNx() {}
-        I SkNx(float val) : fVec(_mm256_set1_ps(val)) {}
-        I SkNx(float a, float b, float c, float d,
-               float e, float f, float g, float h) : fVec(_mm256_setr_ps(a,b,c,d,e,f,g,h)) {}
-
-        I static SkNx Load(const void* ptr) { return _mm256_loadu_ps((const float*)ptr); }
-        I void store(void* ptr) const { _mm256_storeu_ps((float*)ptr, fVec); }
-
-        I SkNx operator+(const SkNx& o) const { return _mm256_add_ps(fVec, o.fVec); }
-        I SkNx operator-(const SkNx& o) const { return _mm256_sub_ps(fVec, o.fVec); }
-        I SkNx operator*(const SkNx& o) const { return _mm256_mul_ps(fVec, o.fVec); }
-        I SkNx operator/(const SkNx& o) const { return _mm256_div_ps(fVec, o.fVec); }
-
-        I SkNx operator==(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_EQ_OQ); }
-        I SkNx operator!=(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_NEQ_OQ); }
-        I SkNx operator <(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_LT_OQ); }
-        I SkNx operator >(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_GT_OQ); }
-        I SkNx operator<=(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_LE_OQ); }
-        I SkNx operator>=(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_GE_OQ); }
-
-        I static SkNx Min(const SkNx& l, const SkNx& r) { return _mm256_min_ps(l.fVec, r.fVec); }
-        I static SkNx Max(const SkNx& l, const SkNx& r) { return _mm256_max_ps(l.fVec, r.fVec); }
-
-        I SkNx   sqrt() const { return _mm256_sqrt_ps (fVec); }
-        I SkNx  rsqrt() const { return _mm256_rsqrt_ps(fVec); }
-        I SkNx invert() const { return _mm256_rcp_ps  (fVec); }
-
-        I float operator[](int k) const {
-            SkASSERT(0 <= k && k < 8);
-            union { __m256 v; float fs[8]; } pun = {fVec};
-            return pun.fs[k&7];
-        }
-
-        I SkNx thenElse(const SkNx& t, const SkNx& e) const {
-            return _mm256_blendv_ps(e.fVec, t.fVec, fVec);
-        }
-
-        __m256 fVec;
-    };
-
-    static I void SkNx_split(const Sk8f& v, Sk4f* lo, Sk4f* hi) {
-        *lo = _mm256_extractf128_ps(v.fVec, 0);
-        *hi = _mm256_extractf128_ps(v.fVec, 1);
-    }
-
-    static I Sk8f SkNx_join(const Sk4f& lo, const Sk4f& hi) {
-        return _mm256_insertf128_ps(_mm256_castps128_ps256(lo.fVec), hi.fVec, 1);
-    }
-
-    static I Sk8f SkNx_fma(const Sk8f& a, const Sk8f& b, const Sk8f& c) {
-        return _mm256_fmadd_ps(a.fVec, b.fVec, c.fVec);
-    }
-
-    template<> /*static*/ I Sk8f SkNx_cast<float>(const Sk8b& src) {
-        return _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(src.fVec));
-    }
-
-    template<> /*static*/ I Sk8f SkNx_cast<float>(const Sk8i& src) {
-        return _mm256_cvtepi32_ps(src.fVec);
-    }
-
-    template<> /*static*/ I Sk8i SkNx_cast<int>(const Sk8f& src) {
-        return _mm256_cvttps_epi32(src.fVec);
-    }
-
-    template<> /*static*/ I Sk8i SkNx_cast<int>(const Sk8h& src) {
-        return _mm256_cvtepu16_epi32(src.fVec);
-    }
-    template<> /*static*/ I Sk8h SkNx_cast<uint16_t>(const Sk8i& src) {
-        __m128i lo = _mm256_extractf128_si256(src.fVec, 0),
-                hi = _mm256_extractf128_si256(src.fVec, 1);
-        return _mm_packus_epi32(lo, hi);
-    }
-
-#undef I
-
-#endif
-
 template<> /*static*/ inline Sk4f SkNx_cast<float, int32_t>(const Sk4i& src) {
     return _mm_cvtepi32_ps(src.fVec);
 }
diff --git a/src/opts/SkOpts_hsw.cpp b/src/opts/SkOpts_hsw.cpp
index 9f724805a6..53e2e5acdd 100644
--- a/src/opts/SkOpts_hsw.cpp
+++ b/src/opts/SkOpts_hsw.cpp
@@ -7,70 +7,9 @@
 
 #include "SkOpts.h"
 
-
 #define SK_OPTS_NS hsw
-#include "SkRasterPipeline_opts.h"
 
 namespace SkOpts {
-    void Init_hsw() {
-
-        run_pipeline = SK_OPTS_NS::run_pipeline;
-
-    #define STAGE(stage)                                                       \
-        body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage;      \
-        tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage##_tail
-
-        STAGE(store_565);
-        STAGE(store_srgb);
-        STAGE(store_f16);
-
-        STAGE(load_s_565);
-        STAGE(load_s_srgb);
-        STAGE(load_s_f16);
-
-        STAGE(load_d_565);
-        STAGE(load_d_srgb);
-        STAGE(load_d_f16);
-
-        STAGE(scale_u8);
-
-        STAGE(lerp_u8);
-        STAGE(lerp_565);
-    #undef STAGE
-
-    #define STAGE(stage) \
-        body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage; \
-        tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage
-
-        STAGE(lerp_constant_float);
-        STAGE(constant_color);
-
-        STAGE(dst);
-        STAGE(dstatop);
-        STAGE(dstin);
-        STAGE(dstout);
-        STAGE(dstover);
-        STAGE(srcatop);
-        STAGE(srcin);
-        STAGE(srcout);
-        STAGE(srcover);
-        STAGE(clear);
-        STAGE(modulate);
-        STAGE(multiply);
-        STAGE(plus_);
-        STAGE(screen);
-        STAGE(xor_);
-        STAGE(colorburn);
-        STAGE(colordodge);
-        STAGE(darken);
-        STAGE(difference);
-        STAGE(exclusion);
-        STAGE(hardlight);
-        STAGE(lighten);
-        STAGE(overlay);
-        STAGE(softlight);
-    #undef STAGE
-
-    }
+    void Init_hsw() { }
 }
 
diff --git a/src/opts/SkOpts_sse41.cpp b/src/opts/SkOpts_sse41.cpp
index d55978c8b7..3a37834c72 100644
--- a/src/opts/SkOpts_sse41.cpp
+++ b/src/opts/SkOpts_sse41.cpp
@@ -21,60 +21,58 @@ namespace SkOpts {
         srcover_srgb_srgb    = sse41::srcover_srgb_srgb;
         blit_row_s32a_opaque = sse41::blit_row_s32a_opaque;
 
-    #define STAGE(stage)                                                       \
-        body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage;      \
-        tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage##_tail
+    #define STAGE(stage, kCallNext) \
+        stages_4  [SkRasterPipeline::stage] = stage_4  <SK_OPTS_NS::stage, kCallNext>; \
+        stages_1_3[SkRasterPipeline::stage] = stage_1_3<SK_OPTS_NS::stage, kCallNext>
 
-        STAGE(store_565);
-        STAGE(store_srgb);
-        STAGE(store_f16);
+        STAGE(store_565 , false);
+        STAGE(store_srgb, false);
+        STAGE(store_f16 , false);
 
-        STAGE(load_s_565);
-        STAGE(load_s_srgb);
-        STAGE(load_s_f16);
+        STAGE(load_s_565 , true);
+        STAGE(load_s_srgb, true);
+        STAGE(load_s_f16 , true);
 
-        STAGE(load_d_565);
-        STAGE(load_d_srgb);
-        STAGE(load_d_f16);
+        STAGE(load_d_565 , true);
+        STAGE(load_d_srgb, true);
+        STAGE(load_d_f16 , true);
 
-        STAGE(scale_u8);
+        STAGE(scale_u8, true);
 
-        STAGE(lerp_u8);
-        STAGE(lerp_565);
-    #undef STAGE
+        STAGE(lerp_u8            , true);
+        STAGE(lerp_565           , true);
+        STAGE(lerp_constant_float, true);
 
-    #define STAGE(stage)                                                  \
-        body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage; \
-        tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage
+        STAGE(constant_color, true);
 
-      // The commented-out stages don't actually benefit from SSE 4.1.
-      // To cut down on code bloat we skip them here, using the identical SSE2 defaults.
+    #undef STAGE
 
-      //STAGE(lerp_constant_float);
-      //STAGE(constant_color);
+    #define STAGE(stage) \
+        stages_4  [SkRasterPipeline::stage] = SK_OPTS_NS::stage; \
+        stages_1_3[SkRasterPipeline::stage] = SK_OPTS_NS::stage
 
-      //STAGE(dst);
-      //STAGE(dstatop);
-      //STAGE(dstin);
-      //STAGE(dstout);
-      //STAGE(dstover);
-      //STAGE(srcatop);
-      //STAGE(srcin);
-      //STAGE(srcout);
-      //STAGE(srcover);
-      //STAGE(clear);
-      //STAGE(modulate);
-      //STAGE(multiply);
-      //STAGE(plus_);
-      //STAGE(screen);
-      //STAGE(xor_);
+        STAGE(dst);
+        STAGE(dstatop);
+        STAGE(dstin);
+        STAGE(dstout);
+        STAGE(dstover);
+        STAGE(srcatop);
+        STAGE(srcin);
+        STAGE(srcout);
+        STAGE(srcover);
+        STAGE(clear);
+        STAGE(modulate);
+        STAGE(multiply);
+        STAGE(plus_);
+        STAGE(screen);
+        STAGE(xor_);
         STAGE(colorburn);
         STAGE(colordodge);
-      //STAGE(darken);
-      //STAGE(difference);
-      //STAGE(exclusion);
+        STAGE(darken);
+        STAGE(difference);
+        STAGE(exclusion);
         STAGE(hardlight);
-      //STAGE(lighten);
+        STAGE(lighten);
         STAGE(overlay);
         STAGE(softlight);
     #undef STAGE
diff --git a/src/opts/SkRasterPipeline_opts.h b/src/opts/SkRasterPipeline_opts.h
index 1d8b04452d..b0e6e1d2f9 100644
--- a/src/opts/SkRasterPipeline_opts.h
+++ b/src/opts/SkRasterPipeline_opts.h
@@ -13,147 +13,129 @@
 #include "SkRasterPipeline.h"
 #include "SkSRGB.h"
 
-using SkNf = SkRasterPipeline::V;
-static constexpr auto N = sizeof(SkNf) / sizeof(float);
-using SkNi = SkNx<N, int>;
-using SkNh = SkNx<N, uint16_t>;
-
-#define SI static inline
-
-#define STAGE(name, kCallNext)                                                             \
-    static SK_ALWAYS_INLINE void name##_kernel(void* ctx, size_t x, size_t tail,           \
-                                               SkNf&  r, SkNf&  g, SkNf&  b, SkNf&  a,     \
-                                               SkNf& dr, SkNf& dg, SkNf& db, SkNf& da);    \
-    SI void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail,         \
-                               SkNf  r, SkNf  g, SkNf  b, SkNf  a,                         \
-                               SkNf dr, SkNf dg, SkNf db, SkNf da) {                       \
-        name##_kernel(st->ctx<void*>(), x,0, r,g,b,a, dr,dg,db,da);                        \
-        if (kCallNext) {                                                                   \
-            st->next(x,tail, r,g,b,a, dr,dg,db,da);                                        \
-        }                                                                                  \
-    }                                                                                      \
-    SI void SK_VECTORCALL name##_tail(SkRasterPipeline::Stage* st, size_t x, size_t tail,  \
-                                      SkNf  r, SkNf  g, SkNf  b, SkNf  a,                  \
-                                      SkNf dr, SkNf dg, SkNf db, SkNf da) {                \
-        name##_kernel(st->ctx<void*>(), x,tail, r,g,b,a, dr,dg,db,da);                     \
-        if (kCallNext) {                                                                   \
-            st->next(x,tail, r,g,b,a, dr,dg,db,da);                                        \
-        }                                                                                  \
-    }                                                                                      \
-    static SK_ALWAYS_INLINE void name##_kernel(void* ctx, size_t x, size_t tail,           \
-                                               SkNf&  r, SkNf&  g, SkNf&  b, SkNf&  a,     \
-                                               SkNf& dr, SkNf& dg, SkNf& db, SkNf& da)
+using Kernel_Sk4f = void(void*, size_t, size_t, Sk4f&, Sk4f&, Sk4f&, Sk4f&,
+                                                Sk4f&, Sk4f&, Sk4f&, Sk4f&);
+
+// These are always static, and we _really_ want them to inline.
+// If you find yourself wanting a non-inline stage, write a SkRasterPipeline::Fn directly.
+#define KERNEL_Sk4f(name)                                                      \
+    static SK_ALWAYS_INLINE void name(void* ctx, size_t x, size_t tail,        \
+                                      Sk4f&  r, Sk4f&  g, Sk4f&  b, Sk4f&  a,  \
+                                      Sk4f& dr, Sk4f& dg, Sk4f& db, Sk4f& da)
+
+
+template <Kernel_Sk4f kernel, bool kCallNext>
+static inline void SK_VECTORCALL stage_4(SkRasterPipeline::Stage* st, size_t x, size_t tail,
+                                         Sk4f  r, Sk4f  g, Sk4f  b, Sk4f  a,
+                                         Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) {
+    // Passing 0 lets the optimizer completely drop any "if (tail) {...}" code in kernel.
+    kernel(st->ctx<void*>(), x,0, r,g,b,a, dr,dg,db,da);
+    if (kCallNext) {
+        st->next(x,tail, r,g,b,a, dr,dg,db,da);  // It's faster to pass t here than 0.
+    }
+}
 
+template <Kernel_Sk4f kernel, bool kCallNext>
+static inline void SK_VECTORCALL stage_1_3(SkRasterPipeline::Stage* st, size_t x, size_t tail,
+                                           Sk4f  r, Sk4f  g, Sk4f  b, Sk4f  a,
+                                           Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) {
+#if defined(__clang__)
+    __builtin_assume(tail > 0);  // This flourish lets Clang compile away any tail==0 code.
+#endif
+    kernel(st->ctx<void*>(), x,tail, r,g,b,a, dr,dg,db,da);
+    if (kCallNext) {
+        st->next(x,tail, r,g,b,a, dr,dg,db,da);
+    }
+}
 
 // Many xfermodes apply the same logic to each channel.
-#define RGBA_XFERMODE(name)                                                                \
-    static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa,              \
-                                               const SkNf& d, const SkNf& da);             \
-    SI void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail,         \
-                               SkNf  r, SkNf  g, SkNf  b, SkNf  a,                         \
-                               SkNf dr, SkNf dg, SkNf db, SkNf da) {                       \
-        r = name##_kernel(r,a,dr,da);                                                      \
-        g = name##_kernel(g,a,dg,da);                                                      \
-        b = name##_kernel(b,a,db,da);                                                      \
-        a = name##_kernel(a,a,da,da);                                                      \
-        st->next(x,tail, r,g,b,a, dr,dg,db,da);                                            \
-    }                                                                                      \
-    static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa,              \
-                                               const SkNf& d, const SkNf& da)
+#define RGBA_XFERMODE_Sk4f(name)                                                       \
+    static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa,          \
+                                               const Sk4f& d, const Sk4f& da);         \
+    static void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail, \
+                                   Sk4f  r, Sk4f  g, Sk4f  b, Sk4f  a,                 \
+                                   Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) {               \
+        r = name##_kernel(r,a,dr,da);                                                  \
+        g = name##_kernel(g,a,dg,da);                                                  \
+        b = name##_kernel(b,a,db,da);                                                  \
+        a = name##_kernel(a,a,da,da);                                                  \
+        st->next(x,tail, r,g,b,a, dr,dg,db,da);                                        \
+    }                                                                                  \
+    static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa,          \
+                                               const Sk4f& d, const Sk4f& da)
 
 // Most of the rest apply the same logic to color channels and use srcover's alpha logic.
-#define RGB_XFERMODE(name)                                                                 \
-    static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa,              \
-                                               const SkNf& d, const SkNf& da);             \
-    SI void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail,         \
-                               SkNf  r, SkNf  g, SkNf  b, SkNf  a,                         \
-                               SkNf dr, SkNf dg, SkNf db, SkNf da) {                       \
-        r = name##_kernel(r,a,dr,da);                                                      \
-        g = name##_kernel(g,a,dg,da);                                                      \
-        b = name##_kernel(b,a,db,da);                                                      \
-        a = a + (da * (1.0f-a));                                                           \
-        st->next(x,tail, r,g,b,a, dr,dg,db,da);                                            \
-    }                                                                                      \
-    static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa,              \
-                                               const SkNf& d, const SkNf& da)
-
+#define RGB_XFERMODE_Sk4f(name)                                                        \
+    static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa,          \
+                                               const Sk4f& d, const Sk4f& da);         \
+    static void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail, \
+                                   Sk4f  r, Sk4f  g, Sk4f  b, Sk4f  a,                 \
+                                   Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) {               \
+        r = name##_kernel(r,a,dr,da);                                                  \
+        g = name##_kernel(g,a,dg,da);                                                  \
+        b = name##_kernel(b,a,db,da);                                                  \
+        a = a + (da * (1.0f-a));                                                       \
+        st->next(x,tail, r,g,b,a, dr,dg,db,da);                                        \
+    }                                                                                  \
+    static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa,          \
+                                               const Sk4f& d, const Sk4f& da)
 
 namespace SK_OPTS_NS {
 
-    SI void run_pipeline(size_t x, size_t n,
-                         void (*vBodyStart)(), SkRasterPipeline::Stage* body,
-                         void (*vTailStart)(), SkRasterPipeline::Stage* tail) {
-        auto bodyStart = (SkRasterPipeline::Fn)vBodyStart,
-             tailStart = (SkRasterPipeline::Fn)vTailStart;
-        SkNf v;  // Fastest to start uninitialized.
-        while (n >= N) {
-            bodyStart(body, x,0, v,v,v,v, v,v,v,v);
-            x += N;
-            n -= N;
-        }
-        if (n > 0) {
-            tailStart(tail, x,n, v,v,v,v, v,v,v,v);
-        }
-    }
-
     // Clamp colors into [0,1] premul (e.g. just before storing back to memory).
-    SI void clamp_01_premul(SkNf& r, SkNf& g, SkNf& b, SkNf& a) {
-        a = SkNf::Max(a, 0.0f);
-        r = SkNf::Max(r, 0.0f);
-        g = SkNf::Max(g, 0.0f);
-        b = SkNf::Max(b, 0.0f);
-
-        a = SkNf::Min(a, 1.0f);
-        r = SkNf::Min(r, a);
-        g = SkNf::Min(g, a);
-        b = SkNf::Min(b, a);
+    static void clamp_01_premul(Sk4f& r, Sk4f& g, Sk4f& b, Sk4f& a) {
+        a = Sk4f::Max(a, 0.0f);
+        r = Sk4f::Max(r, 0.0f);
+        g = Sk4f::Max(g, 0.0f);
+        b = Sk4f::Max(b, 0.0f);
+
+        a = Sk4f::Min(a, 1.0f);
+        r = Sk4f::Min(r, a);
+        g = Sk4f::Min(g, a);
+        b = Sk4f::Min(b, a);
     }
 
-    SI SkNf inv(const SkNf& x) { return 1.0f - x; }
+    static Sk4f inv(const Sk4f& x) { return 1.0f - x; }
 
-    SI SkNf lerp(const SkNf& from, const SkNf& to, const SkNf& cov) {
-        return SkNx_fma(to-from, cov, from);
+    static Sk4f lerp(const Sk4f& from, const Sk4f& to, const Sk4f& cov) {
+        return from + (to-from)*cov;
     }
 
     template <typename T>
-    SI SkNx<N,T> load_tail(size_t tail, const T* src) {
-        // TODO: better tail, maskload for 32- and 64-bit T
-        T buf[N] = {0};
+    static SkNx<4,T> load_tail(size_t tail, const T* src) {
         if (tail) {
-            memcpy(buf, src, tail*sizeof(T));
-            src = buf;
+           return SkNx<4,T>(src[0], (tail>1 ? src[1] : 0), (tail>2 ? src[2] : 0), 0);
         }
-        return SkNx<N,T>::Load(src);
+        return SkNx<4,T>::Load(src);
     }
 
     template <typename T>
-    SI void store_tail(size_t tail, const SkNx<N,T>& v, T* dst) {
-        // TODO: better tail, maskstore for 32- and 64-bit T
-        T buf[N] = {0};
-        v.store(tail ? buf : dst);
-        if (tail) {
-            memcpy(dst, buf, tail*sizeof(T));
+    static void store_tail(size_t tail, const SkNx<4,T>& v, T* dst) {
+        switch(tail) {
+            case 0: return v.store(dst);
+            case 3: dst[2] = v[2];
+            case 2: dst[1] = v[1];
+            case 1: dst[0] = v[0];
         }
     }
 
-    SI void from_565(const SkNh& _565, SkNf* r, SkNf* g, SkNf* b) {
-        auto _32_bit = SkNx_cast<int>(_565);
+    static void from_565(const Sk4h& _565, Sk4f* r, Sk4f* g, Sk4f* b) {
+        Sk4i _32_bit = SkNx_cast<int>(_565);
 
         *r = SkNx_cast<float>(_32_bit & SK_R16_MASK_IN_PLACE) * (1.0f / SK_R16_MASK_IN_PLACE);
         *g = SkNx_cast<float>(_32_bit & SK_G16_MASK_IN_PLACE) * (1.0f / SK_G16_MASK_IN_PLACE);
         *b = SkNx_cast<float>(_32_bit & SK_B16_MASK_IN_PLACE) * (1.0f / SK_B16_MASK_IN_PLACE);
     }
 
-    SI SkNh to_565(const SkNf& r, const SkNf& g, const SkNf& b) {
-        return SkNx_cast<uint16_t>( SkNx_cast<int>(r * SK_R16_MASK + 0.5f) << SK_R16_SHIFT
-                                  | SkNx_cast<int>(g * SK_G16_MASK + 0.5f) << SK_G16_SHIFT
-                                  | SkNx_cast<int>(b * SK_B16_MASK + 0.5f) << SK_B16_SHIFT);
+    static Sk4h to_565(const Sk4f& r, const Sk4f& g, const Sk4f& b) {
+        return SkNx_cast<uint16_t>( Sk4f_round(r * SK_R16_MASK) << SK_R16_SHIFT
+                                  | Sk4f_round(g * SK_G16_MASK) << SK_G16_SHIFT
+                                  | Sk4f_round(b * SK_B16_MASK) << SK_B16_SHIFT);
     }
 
-    STAGE(just_return, false) { }
 
     // The default shader produces a constant color (from the SkPaint).
-    STAGE(constant_color, true) {
+    KERNEL_Sk4f(constant_color) {
         auto color = (const SkPM4f*)ctx;
         r = color->r();
         g = color->g();
@@ -162,8 +144,8 @@ namespace SK_OPTS_NS {
     }
 
     // s' = d(1-c) + sc, for a constant c.
-    STAGE(lerp_constant_float, true) {
-        SkNf c = *(const float*)ctx;
+    KERNEL_Sk4f(lerp_constant_float) {
+        Sk4f c = *(const float*)ctx;
 
         r = lerp(dr, r, c);
         g = lerp(dg, g, c);
@@ -172,10 +154,10 @@ namespace SK_OPTS_NS {
     }
 
     // s' = sc for 8-bit c.
-    STAGE(scale_u8, true) {
+    KERNEL_Sk4f(scale_u8) {
         auto ptr = (const uint8_t*)ctx + x;
 
-        SkNf c = SkNx_cast<float>(load_tail(tail, ptr)) * (1/255.0f);
+        Sk4f c = SkNx_cast<float>(load_tail(tail, ptr)) * (1/255.0f);
         r = r*c;
         g = g*c;
         b = b*c;
@@ -183,10 +165,10 @@ namespace SK_OPTS_NS {
     }
 
     // s' = d(1-c) + sc for 8-bit c.
-    STAGE(lerp_u8, true) {
+    KERNEL_Sk4f(lerp_u8) {
         auto ptr = (const uint8_t*)ctx + x;
 
-        SkNf c = SkNx_cast<float>(load_tail(tail, ptr)) * (1/255.0f);
+        Sk4f c = SkNx_cast<float>(load_tail(tail, ptr)) * (1/255.0f);
         r = lerp(dr, r, c);
         g = lerp(dg, g, c);
         b = lerp(db, b, c);
@@ -194,9 +176,9 @@ namespace SK_OPTS_NS {
     }
 
     // s' = d(1-c) + sc for 565 c.
-    STAGE(lerp_565, true) {
+    KERNEL_Sk4f(lerp_565) {
         auto ptr = (const uint16_t*)ctx + x;
-        SkNf cr, cg, cb;
+        Sk4f cr, cg, cb;
         from_565(load_tail(tail, ptr), &cr, &cg, &cb);
 
         r = lerp(dr, r, cr);
@@ -205,145 +187,155 @@ namespace SK_OPTS_NS {
         a = 1.0f;
     }
 
-    STAGE(load_d_565, true) {
+    KERNEL_Sk4f(load_d_565) {
         auto ptr = (const uint16_t*)ctx + x;
         from_565(load_tail(tail, ptr), &dr,&dg,&db);
         da = 1.0f;
     }
 
-    STAGE(load_s_565, true) {
+    KERNEL_Sk4f(load_s_565) {
         auto ptr = (const uint16_t*)ctx + x;
         from_565(load_tail(tail, ptr), &r,&g,&b);
         a = 1.0f;
     }
 
-    STAGE(store_565, false) {
+    KERNEL_Sk4f(store_565) {
         clamp_01_premul(r,g,b,a);
         auto ptr = (uint16_t*)ctx + x;
         store_tail(tail, to_565(r,g,b), ptr);
     }
 
-    STAGE(load_d_f16, true) {
+    KERNEL_Sk4f(load_d_f16) {
         auto ptr = (const uint64_t*)ctx + x;
 
-        uint64_t buf[N] = {0};
         if (tail) {
-            memcpy(buf, ptr, tail*sizeof(uint64_t));
-            ptr = buf;
+            auto p0 =          SkHalfToFloat_finite_ftz(ptr[0])          ,
+                 p1 = tail>1 ? SkHalfToFloat_finite_ftz(ptr[1]) : Sk4f{0},
+                 p2 = tail>2 ? SkHalfToFloat_finite_ftz(ptr[2]) : Sk4f{0};
+            dr = { p0[0],p1[0],p2[0],0 };
+            dg = { p0[1],p1[1],p2[1],0 };
+            db = { p0[2],p1[2],p2[2],0 };
+            da = { p0[3],p1[3],p2[3],0 };
+            return;
         }
 
-        SkNh rh, gh, bh, ah;
-        SkNh::Load4(ptr, &rh, &gh, &bh, &ah);
+        Sk4h rh, gh, bh, ah;
+        Sk4h::Load4(ptr, &rh, &gh, &bh, &ah);
         dr = SkHalfToFloat_finite_ftz(rh);
         dg = SkHalfToFloat_finite_ftz(gh);
         db = SkHalfToFloat_finite_ftz(bh);
         da = SkHalfToFloat_finite_ftz(ah);
     }
 
-    STAGE(load_s_f16, true) {
+    KERNEL_Sk4f(load_s_f16) {
         auto ptr = (const uint64_t*)ctx + x;
 
-        uint64_t buf[N] = {0};
         if (tail) {
-            memcpy(buf, ptr, tail*sizeof(uint64_t));
-            ptr = buf;
+            auto p0 =          SkHalfToFloat_finite_ftz(ptr[0])          ,
+                 p1 = tail>1 ? SkHalfToFloat_finite_ftz(ptr[1]) : Sk4f{0},
+                 p2 = tail>2 ? SkHalfToFloat_finite_ftz(ptr[2]) : Sk4f{0};
+            r = { p0[0],p1[0],p2[0],0 };
+            g = { p0[1],p1[1],p2[1],0 };
+            b = { p0[2],p1[2],p2[2],0 };
+            a = { p0[3],p1[3],p2[3],0 };
+            return;
         }
 
-        SkNh rh, gh, bh, ah;
-        SkNh::Load4(ptr, &rh, &gh, &bh, &ah);
+        Sk4h rh, gh, bh, ah;
+        Sk4h::Load4(ptr, &rh, &gh, &bh, &ah);
         r = SkHalfToFloat_finite_ftz(rh);
         g = SkHalfToFloat_finite_ftz(gh);
         b = SkHalfToFloat_finite_ftz(bh);
         a = SkHalfToFloat_finite_ftz(ah);
     }
 
-    STAGE(store_f16, false) {
+    KERNEL_Sk4f(store_f16) {
         clamp_01_premul(r,g,b,a);
         auto ptr = (uint64_t*)ctx + x;
 
-        uint64_t buf[N] = {0};
-        SkNh::Store4(tail ? buf : ptr, SkFloatToHalf_finite_ftz(r),
-                                       SkFloatToHalf_finite_ftz(g),
-                                       SkFloatToHalf_finite_ftz(b),
-                                       SkFloatToHalf_finite_ftz(a));
-        if (tail) {
-            memcpy(ptr, buf, tail*sizeof(uint64_t));
+        switch (tail) {
+            case 0: return Sk4h::Store4(ptr, SkFloatToHalf_finite_ftz(r),
+                                             SkFloatToHalf_finite_ftz(g),
+                                             SkFloatToHalf_finite_ftz(b),
+                                             SkFloatToHalf_finite_ftz(a));
+
+            case 3: SkFloatToHalf_finite_ftz({r[2], g[2], b[2], a[2]}).store(ptr+2);
+            case 2: SkFloatToHalf_finite_ftz({r[1], g[1], b[1], a[1]}).store(ptr+1);
+            case 1: SkFloatToHalf_finite_ftz({r[0], g[0], b[0], a[0]}).store(ptr+0);
         }
     }
 
 
     // Load 8-bit SkPMColor-order sRGB.
-    STAGE(load_d_srgb, true) {
+    KERNEL_Sk4f(load_d_srgb) {
         auto ptr = (const uint32_t*)ctx + x;
 
-        auto px = load_tail(tail, ptr);
-        auto to_int = [](const SkNx<N, uint32_t>& v) { return SkNi::Load(&v); };
-        dr =    sk_linear_from_srgb_math(to_int((px >> SK_R32_SHIFT) & 0xff));
-        dg =    sk_linear_from_srgb_math(to_int((px >> SK_G32_SHIFT) & 0xff));
-        db =    sk_linear_from_srgb_math(to_int((px >> SK_B32_SHIFT) & 0xff));
-        da = (1/255.0f)*SkNx_cast<float>(to_int( px >> SK_A32_SHIFT        ));
+        auto px = load_tail(tail, (const int*)ptr);
+        dr =    sk_linear_from_srgb_math((px >> SK_R32_SHIFT) & 0xff);
+        dg =    sk_linear_from_srgb_math((px >> SK_G32_SHIFT) & 0xff);
+        db =    sk_linear_from_srgb_math((px >> SK_B32_SHIFT) & 0xff);
+        da = (1/255.0f)*SkNx_cast<float>((px >> SK_A32_SHIFT) & 0xff);
     }
 
-    STAGE(load_s_srgb, true) {
+    KERNEL_Sk4f(load_s_srgb) {
         auto ptr = (const uint32_t*)ctx + x;
 
-        auto px = load_tail(tail, ptr);
-        auto to_int = [](const SkNx<N, uint32_t>& v) { return SkNi::Load(&v); };
-        r =    sk_linear_from_srgb_math(to_int((px >> SK_R32_SHIFT) & 0xff));
-        g =    sk_linear_from_srgb_math(to_int((px >> SK_G32_SHIFT) & 0xff));
-        b =    sk_linear_from_srgb_math(to_int((px >> SK_B32_SHIFT) & 0xff));
-        a = (1/255.0f)*SkNx_cast<float>(to_int( px >> SK_A32_SHIFT        ));
+        auto px = load_tail(tail, (const int*)ptr);
+        r =    sk_linear_from_srgb_math((px >> SK_R32_SHIFT) & 0xff);
+        g =    sk_linear_from_srgb_math((px >> SK_G32_SHIFT) & 0xff);
+        b =    sk_linear_from_srgb_math((px >> SK_B32_SHIFT) & 0xff);
+        a = (1/255.0f)*SkNx_cast<float>((px >> SK_A32_SHIFT) & 0xff);
     }
 
-    STAGE(store_srgb, false) {
+    KERNEL_Sk4f(store_srgb) {
         clamp_01_premul(r,g,b,a);
         auto ptr = (uint32_t*)ctx + x;
-        store_tail(tail, (      sk_linear_to_srgb_noclamp(r) << SK_R32_SHIFT
-                         |      sk_linear_to_srgb_noclamp(g) << SK_G32_SHIFT
-                         |      sk_linear_to_srgb_noclamp(b) << SK_B32_SHIFT
-                         | SkNx_cast<int>(255.0f * a + 0.5f) << SK_A32_SHIFT ), (int*)ptr);
+        store_tail(tail, ( sk_linear_to_srgb_noclamp(r) << SK_R32_SHIFT
+                         | sk_linear_to_srgb_noclamp(g) << SK_G32_SHIFT
+                         | sk_linear_to_srgb_noclamp(b) << SK_B32_SHIFT
+                         |       Sk4f_round(255.0f * a) << SK_A32_SHIFT), (int*)ptr);
     }
 
-    RGBA_XFERMODE(clear)    { return 0.0f; }
-  //RGBA_XFERMODE(src)      { return s; }   // This would be a no-op stage, so we just omit it.
-    RGBA_XFERMODE(dst)      { return d; }
-
-    RGBA_XFERMODE(srcatop)  { return s*da + d*inv(sa); }
-    RGBA_XFERMODE(srcin)    { return s * da; }
-    RGBA_XFERMODE(srcout)   { return s * inv(da); }
-    RGBA_XFERMODE(srcover)  { return SkNx_fma(d, inv(sa), s); }
-    RGBA_XFERMODE(dstatop)  { return srcatop_kernel(d,da,s,sa); }
-    RGBA_XFERMODE(dstin)    { return srcin_kernel  (d,da,s,sa); }
-    RGBA_XFERMODE(dstout)   { return srcout_kernel (d,da,s,sa); }
-    RGBA_XFERMODE(dstover)  { return srcover_kernel(d,da,s,sa); }
-
-    RGBA_XFERMODE(modulate) { return s*d; }
-    RGBA_XFERMODE(multiply) { return s*inv(da) + d*inv(sa) + s*d; }
-    RGBA_XFERMODE(plus_)    { return s + d; }
-    RGBA_XFERMODE(screen)   { return s + d - s*d; }
-    RGBA_XFERMODE(xor_)     { return s*inv(da) + d*inv(sa); }
-
-    RGB_XFERMODE(colorburn) {
+    RGBA_XFERMODE_Sk4f(clear)    { return 0.0f; }
+  //RGBA_XFERMODE_Sk4f(src)      { return s; }   // This would be a no-op stage, so we just omit it.
+    RGBA_XFERMODE_Sk4f(dst)      { return d; }
+
+    RGBA_XFERMODE_Sk4f(srcatop)  { return s*da + d*inv(sa); }
+    RGBA_XFERMODE_Sk4f(srcin)    { return s * da; }
+    RGBA_XFERMODE_Sk4f(srcout)   { return s * inv(da); }
+    RGBA_XFERMODE_Sk4f(srcover)  { return s + inv(sa)*d; }
+    RGBA_XFERMODE_Sk4f(dstatop)  { return srcatop_kernel(d,da,s,sa); }
+    RGBA_XFERMODE_Sk4f(dstin)    { return srcin_kernel  (d,da,s,sa); }
+    RGBA_XFERMODE_Sk4f(dstout)   { return srcout_kernel (d,da,s,sa); }
+    RGBA_XFERMODE_Sk4f(dstover)  { return srcover_kernel(d,da,s,sa); }
+
+    RGBA_XFERMODE_Sk4f(modulate) { return s*d; }
+    RGBA_XFERMODE_Sk4f(multiply) { return s*inv(da) + d*inv(sa) + s*d; }
+    RGBA_XFERMODE_Sk4f(plus_)    { return s + d; }
+    RGBA_XFERMODE_Sk4f(screen)   { return s + d - s*d; }
+    RGBA_XFERMODE_Sk4f(xor_)     { return s*inv(da) + d*inv(sa); }
+
+    RGB_XFERMODE_Sk4f(colorburn) {
         return (d == da  ).thenElse(d + s*inv(da),
                (s == 0.0f).thenElse(s + d*inv(sa),
-                                    sa*(da - SkNf::Min(da, (da-d)*sa/s)) + s*inv(da) + d*inv(sa)));
+                                    sa*(da - Sk4f::Min(da, (da-d)*sa/s)) + s*inv(da) + d*inv(sa)));
     }
-    RGB_XFERMODE(colordodge) {
+    RGB_XFERMODE_Sk4f(colordodge) {
         return (d == 0.0f).thenElse(d + s*inv(da),
                (s == sa  ).thenElse(s + d*inv(sa),
-                                    sa*SkNf::Min(da, (d*sa)/(sa - s)) + s*inv(da) + d*inv(sa)));
+                                    sa*Sk4f::Min(da, (d*sa)/(sa - s)) + s*inv(da) + d*inv(sa)));
     }
-    RGB_XFERMODE(darken)     { return s + d - SkNf::Max(s*da, d*sa); }
-    RGB_XFERMODE(difference) { return s + d - 2.0f*SkNf::Min(s*da,d*sa); }
-    RGB_XFERMODE(exclusion)  { return s + d - 2.0f*s*d; }
-    RGB_XFERMODE(hardlight) {
+    RGB_XFERMODE_Sk4f(darken)     { return s + d - Sk4f::Max(s*da, d*sa); }
+    RGB_XFERMODE_Sk4f(difference) { return s + d - 2.0f*Sk4f::Min(s*da,d*sa); }
+    RGB_XFERMODE_Sk4f(exclusion)  { return s + d - 2.0f*s*d; }
+    RGB_XFERMODE_Sk4f(hardlight) {
         return s*inv(da) + d*inv(sa)
              + (2.0f*s <= sa).thenElse(2.0f*s*d, sa*da - 2.0f*(da-d)*(sa-s));
     }
-    RGB_XFERMODE(lighten) { return s + d - SkNf::Min(s*da, d*sa); }
-    RGB_XFERMODE(overlay) { return hardlight_kernel(d,da,s,sa); }
-    RGB_XFERMODE(softlight) {
-        SkNf m  = (da > 0.0f).thenElse(d / da, 0.0f),
+    RGB_XFERMODE_Sk4f(lighten) { return s + d - Sk4f::Min(s*da, d*sa); }
+    RGB_XFERMODE_Sk4f(overlay) { return hardlight_kernel(d,da,s,sa); }
+    RGB_XFERMODE_Sk4f(softlight) {
+        Sk4f m  = (da > 0.0f).thenElse(d / da, 0.0f),
              s2 = 2.0f*s,
              m4 = 4.0f*m;
 
@@ -351,7 +343,7 @@ namespace SK_OPTS_NS {
         //    1. dark src?
         //    2. light src, dark dst?
         //    3. light src, light dst?
-        SkNf darkSrc = d*(sa + (s2 - sa)*(1.0f - m)),     // Used in case 1.
+        Sk4f darkSrc = d*(sa + (s2 - sa)*(1.0f - m)),     // Used in case 1.
              darkDst = (m4*m4 + m4)*(m - 1.0f) + 7.0f*m,  // Used in case 2.
              liteDst = m.rsqrt().invert() - m,            // Used in case 3.
              liteSrc = d*sa + da*(s2 - sa) * (4.0f*d <= da).thenElse(darkDst, liteDst);  // 2 or 3?
@@ -359,9 +351,8 @@ namespace SK_OPTS_NS {
     }
 }
 
-#undef SI
-#undef STAGE
-#undef RGBA_XFERMODE
-#undef RGB_XFERMODE
+#undef KERNEL_Sk4f
+#undef RGB_XFERMODE_Sk4f
+#undef RGB_XFERMODE_Sk4f
 
 #endif//SkRasterPipeline_opts_DEFINED