From 42f4b42e8311f168aeeadd939b476c05b329500e Mon Sep 17 00:00:00 2001 From: Mike Klein Date: Mon, 10 Oct 2016 14:23:37 +0000 Subject: Revert "SkRasterPipeline: 8x pipelines, attempt 2" This reverts commit Id0ba250037e271a9475fe2f0989d64f0aa909bae. crbug.com/654213 Looks like Chrome Canary's picking up Haswell code on non-Haswell machines. Change-Id: I16f976da24db86d5c99636c472ffad56db213a2a Reviewed-on: https://skia-review.googlesource.com/3108 Commit-Queue: Mike Klein Reviewed-by: Mike Klein --- src/opts/SkNx_sse.h | 223 +---------------------- src/opts/SkOpts_hsw.cpp | 63 +------ src/opts/SkOpts_sse41.cpp | 82 +++++---- src/opts/SkRasterPipeline_opts.h | 379 +++++++++++++++++++-------------------- 4 files changed, 227 insertions(+), 520 deletions(-) (limited to 'src/opts') diff --git a/src/opts/SkNx_sse.h b/src/opts/SkNx_sse.h index 4546280f02..66b5f0e9b3 100644 --- a/src/opts/SkNx_sse.h +++ b/src/opts/SkNx_sse.h @@ -302,47 +302,12 @@ public: SkNx() {} SkNx(uint16_t val) : fVec(_mm_set1_epi16(val)) {} + static SkNx Load(const void* ptr) { return _mm_loadu_si128((const __m128i*)ptr); } SkNx(uint16_t a, uint16_t b, uint16_t c, uint16_t d, uint16_t e, uint16_t f, uint16_t g, uint16_t h) : fVec(_mm_setr_epi16(a,b,c,d,e,f,g,h)) {} - static SkNx Load(const void* ptr) { return _mm_loadu_si128((const __m128i*)ptr); } void store(void* ptr) const { _mm_storeu_si128((__m128i*)ptr, fVec); } - static void Load4(const void* ptr, SkNx* r, SkNx* g, SkNx* b, SkNx* a) { - // TODO: AVX2 version - __m128i _01 = _mm_loadu_si128(((__m128i*)ptr) + 0), - _23 = _mm_loadu_si128(((__m128i*)ptr) + 1), - _45 = _mm_loadu_si128(((__m128i*)ptr) + 2), - _67 = _mm_loadu_si128(((__m128i*)ptr) + 3); - - __m128i _02 = _mm_unpacklo_epi16(_01, _23), // r0 r2 g0 g2 b0 b2 a0 a2 - _13 = _mm_unpackhi_epi16(_01, _23), // r1 r3 g1 g3 b1 b3 a1 a3 - _46 = _mm_unpacklo_epi16(_45, _67), - _57 = _mm_unpackhi_epi16(_45, _67); - - __m128i rg0123 = _mm_unpacklo_epi16(_02, _13), // r0 r1 r2 r3 g0 g1 g2 g3 - ba0123 = _mm_unpackhi_epi16(_02, _13), // b0 b1 b2 b3 a0 a1 a2 a3 - rg4567 = _mm_unpacklo_epi16(_46, _57), - ba4567 = _mm_unpackhi_epi16(_46, _57); - - *r = _mm_unpacklo_epi64(rg0123, rg4567); - *g = _mm_unpackhi_epi64(rg0123, rg4567); - *b = _mm_unpacklo_epi64(ba0123, ba4567); - *a = _mm_unpackhi_epi64(ba0123, ba4567); - } - static void Store4(void* ptr, const SkNx& r, const SkNx& g, const SkNx& b, const SkNx& a) { - // TODO: AVX2 version - __m128i rg0123 = _mm_unpacklo_epi16(r.fVec, g.fVec), // r0 g0 r1 g1 r2 g2 r3 g3 - rg4567 = _mm_unpackhi_epi16(r.fVec, g.fVec), // r4 g4 r5 g5 r6 g6 r7 g7 - ba0123 = _mm_unpacklo_epi16(b.fVec, a.fVec), - ba4567 = _mm_unpackhi_epi16(b.fVec, a.fVec); - - _mm_storeu_si128((__m128i*)ptr + 0, _mm_unpacklo_epi32(rg0123, ba0123)); - _mm_storeu_si128((__m128i*)ptr + 1, _mm_unpackhi_epi32(rg0123, ba0123)); - _mm_storeu_si128((__m128i*)ptr + 2, _mm_unpacklo_epi32(rg4567, ba4567)); - _mm_storeu_si128((__m128i*)ptr + 3, _mm_unpackhi_epi32(rg4567, ba4567)); - } - SkNx operator + (const SkNx& o) const { return _mm_add_epi16(fVec, o.fVec); } SkNx operator - (const SkNx& o) const { return _mm_sub_epi16(fVec, o.fVec); } SkNx operator * (const SkNx& o) const { return _mm_mullo_epi16(fVec, o.fVec); } @@ -438,192 +403,6 @@ public: __m128i fVec; }; -#if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_AVX2 - -// There are two different SkNx<8, uint8_t>, SkNx<8, int32_t>, SkNx<8, uint32_t>, SkNx<8, float>: -// - the default paired SkNx<4, ...> versions used without AVX2 -// - the native AVX2 versions. -// It is important that we don't call methods for one from the other. -// Usually these methods inline, but they don't always in Debug builds. -// For now, try to fix this by marking all the AVX2 versions as always-inline. -// We may want or need to extend this strategy to all SkNx methods. -#define I SK_ALWAYS_INLINE - - template <> - class SkNx<8, uint8_t> { - public: - I SkNx(const __m128i& vec) : fVec(vec) {} - - I SkNx() {} - I SkNx(uint8_t v) : fVec(_mm_set1_epi8(v)) {} - I SkNx(uint8_t a, uint8_t b, uint8_t c, uint8_t d, - uint8_t e, uint8_t f, uint8_t g, uint8_t h) - : fVec(_mm_setr_epi8(a,b,c,d, e,f,g,h, 0,0,0,0, 0,0,0,0)) {} - - - I static SkNx Load(const void* ptr) { return _mm_loadl_epi64((const __m128i*)ptr); } - I void store(void* ptr) const { _mm_storel_epi64((__m128i*)ptr, fVec); } - - I uint8_t operator[](int k) const { - SkASSERT(0 <= k && k < 8); - union { __m128i v; uint8_t us[16]; } pun = {fVec}; - return pun.us[k&7]; - } - - __m128i fVec; - }; - - template <> - class SkNx<8, int32_t> { - public: - I SkNx(const __m256i& vec) : fVec(vec) {} - - I SkNx() {} - I SkNx(int32_t v) : fVec(_mm256_set1_epi32(v)) {} - I SkNx(int32_t a, int32_t b, int32_t c, int32_t d, - int32_t e, int32_t f, int32_t g, int32_t h) - : fVec(_mm256_setr_epi32(a,b,c,d, e,f,g,h)) {} - - I static SkNx Load(const void* ptr) { return _mm256_loadu_si256((const __m256i*)ptr); } - I void store(void* ptr) const { _mm256_storeu_si256((__m256i*)ptr, fVec); } - - I SkNx operator + (const SkNx& o) const { return _mm256_add_epi32(fVec, o.fVec); } - I SkNx operator - (const SkNx& o) const { return _mm256_sub_epi32(fVec, o.fVec); } - I SkNx operator * (const SkNx& o) const { return _mm256_mullo_epi32(fVec, o.fVec); } - - I SkNx operator & (const SkNx& o) const { return _mm256_and_si256(fVec, o.fVec); } - I SkNx operator | (const SkNx& o) const { return _mm256_or_si256(fVec, o.fVec); } - I SkNx operator ^ (const SkNx& o) const { return _mm256_xor_si256(fVec, o.fVec); } - - I SkNx operator << (int bits) const { return _mm256_slli_epi32(fVec, bits); } - I SkNx operator >> (int bits) const { return _mm256_srai_epi32(fVec, bits); } - - I int32_t operator[](int k) const { - SkASSERT(0 <= k && k < 8); - union { __m256i v; int32_t is[8]; } pun = {fVec}; - return pun.is[k&7]; - } - - __m256i fVec; - }; - - template <> - class SkNx<8, uint32_t> { - public: - I SkNx(const __m256i& vec) : fVec(vec) {} - - I SkNx() {} - I SkNx(uint32_t v) : fVec(_mm256_set1_epi32(v)) {} - I SkNx(uint32_t a, uint32_t b, uint32_t c, uint32_t d, - uint32_t e, uint32_t f, uint32_t g, uint32_t h) - : fVec(_mm256_setr_epi32(a,b,c,d, e,f,g,h)) {} - - I static SkNx Load(const void* ptr) { return _mm256_loadu_si256((const __m256i*)ptr); } - I void store(void* ptr) const { _mm256_storeu_si256((__m256i*)ptr, fVec); } - - I SkNx operator + (const SkNx& o) const { return _mm256_add_epi32(fVec, o.fVec); } - I SkNx operator - (const SkNx& o) const { return _mm256_sub_epi32(fVec, o.fVec); } - I SkNx operator * (const SkNx& o) const { return _mm256_mullo_epi32(fVec, o.fVec); } - - I SkNx operator & (const SkNx& o) const { return _mm256_and_si256(fVec, o.fVec); } - I SkNx operator | (const SkNx& o) const { return _mm256_or_si256(fVec, o.fVec); } - I SkNx operator ^ (const SkNx& o) const { return _mm256_xor_si256(fVec, o.fVec); } - - I SkNx operator << (int bits) const { return _mm256_slli_epi32(fVec, bits); } - I SkNx operator >> (int bits) const { return _mm256_srli_epi32(fVec, bits); } - - I uint32_t operator[](int k) const { - SkASSERT(0 <= k && k < 8); - union { __m256i v; uint32_t us[8]; } pun = {fVec}; - return pun.us[k&7]; - } - - __m256i fVec; - }; - - template <> - class SkNx<8, float> { - public: - I SkNx(const __m256& vec) : fVec(vec) {} - - I SkNx() {} - I SkNx(float val) : fVec(_mm256_set1_ps(val)) {} - I SkNx(float a, float b, float c, float d, - float e, float f, float g, float h) : fVec(_mm256_setr_ps(a,b,c,d,e,f,g,h)) {} - - I static SkNx Load(const void* ptr) { return _mm256_loadu_ps((const float*)ptr); } - I void store(void* ptr) const { _mm256_storeu_ps((float*)ptr, fVec); } - - I SkNx operator+(const SkNx& o) const { return _mm256_add_ps(fVec, o.fVec); } - I SkNx operator-(const SkNx& o) const { return _mm256_sub_ps(fVec, o.fVec); } - I SkNx operator*(const SkNx& o) const { return _mm256_mul_ps(fVec, o.fVec); } - I SkNx operator/(const SkNx& o) const { return _mm256_div_ps(fVec, o.fVec); } - - I SkNx operator==(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_EQ_OQ); } - I SkNx operator!=(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_NEQ_OQ); } - I SkNx operator <(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_LT_OQ); } - I SkNx operator >(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_GT_OQ); } - I SkNx operator<=(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_LE_OQ); } - I SkNx operator>=(const SkNx& o) const { return _mm256_cmp_ps(fVec, o.fVec, _CMP_GE_OQ); } - - I static SkNx Min(const SkNx& l, const SkNx& r) { return _mm256_min_ps(l.fVec, r.fVec); } - I static SkNx Max(const SkNx& l, const SkNx& r) { return _mm256_max_ps(l.fVec, r.fVec); } - - I SkNx sqrt() const { return _mm256_sqrt_ps (fVec); } - I SkNx rsqrt() const { return _mm256_rsqrt_ps(fVec); } - I SkNx invert() const { return _mm256_rcp_ps (fVec); } - - I float operator[](int k) const { - SkASSERT(0 <= k && k < 8); - union { __m256 v; float fs[8]; } pun = {fVec}; - return pun.fs[k&7]; - } - - I SkNx thenElse(const SkNx& t, const SkNx& e) const { - return _mm256_blendv_ps(e.fVec, t.fVec, fVec); - } - - __m256 fVec; - }; - - static I void SkNx_split(const Sk8f& v, Sk4f* lo, Sk4f* hi) { - *lo = _mm256_extractf128_ps(v.fVec, 0); - *hi = _mm256_extractf128_ps(v.fVec, 1); - } - - static I Sk8f SkNx_join(const Sk4f& lo, const Sk4f& hi) { - return _mm256_insertf128_ps(_mm256_castps128_ps256(lo.fVec), hi.fVec, 1); - } - - static I Sk8f SkNx_fma(const Sk8f& a, const Sk8f& b, const Sk8f& c) { - return _mm256_fmadd_ps(a.fVec, b.fVec, c.fVec); - } - - template<> /*static*/ I Sk8f SkNx_cast(const Sk8b& src) { - return _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(src.fVec)); - } - - template<> /*static*/ I Sk8f SkNx_cast(const Sk8i& src) { - return _mm256_cvtepi32_ps(src.fVec); - } - - template<> /*static*/ I Sk8i SkNx_cast(const Sk8f& src) { - return _mm256_cvttps_epi32(src.fVec); - } - - template<> /*static*/ I Sk8i SkNx_cast(const Sk8h& src) { - return _mm256_cvtepu16_epi32(src.fVec); - } - template<> /*static*/ I Sk8h SkNx_cast(const Sk8i& src) { - __m128i lo = _mm256_extractf128_si256(src.fVec, 0), - hi = _mm256_extractf128_si256(src.fVec, 1); - return _mm_packus_epi32(lo, hi); - } - -#undef I - -#endif - template<> /*static*/ inline Sk4f SkNx_cast(const Sk4i& src) { return _mm_cvtepi32_ps(src.fVec); } diff --git a/src/opts/SkOpts_hsw.cpp b/src/opts/SkOpts_hsw.cpp index 9f724805a6..53e2e5acdd 100644 --- a/src/opts/SkOpts_hsw.cpp +++ b/src/opts/SkOpts_hsw.cpp @@ -7,70 +7,9 @@ #include "SkOpts.h" - #define SK_OPTS_NS hsw -#include "SkRasterPipeline_opts.h" namespace SkOpts { - void Init_hsw() { - - run_pipeline = SK_OPTS_NS::run_pipeline; - - #define STAGE(stage) \ - body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage; \ - tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage##_tail - - STAGE(store_565); - STAGE(store_srgb); - STAGE(store_f16); - - STAGE(load_s_565); - STAGE(load_s_srgb); - STAGE(load_s_f16); - - STAGE(load_d_565); - STAGE(load_d_srgb); - STAGE(load_d_f16); - - STAGE(scale_u8); - - STAGE(lerp_u8); - STAGE(lerp_565); - #undef STAGE - - #define STAGE(stage) \ - body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage; \ - tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage - - STAGE(lerp_constant_float); - STAGE(constant_color); - - STAGE(dst); - STAGE(dstatop); - STAGE(dstin); - STAGE(dstout); - STAGE(dstover); - STAGE(srcatop); - STAGE(srcin); - STAGE(srcout); - STAGE(srcover); - STAGE(clear); - STAGE(modulate); - STAGE(multiply); - STAGE(plus_); - STAGE(screen); - STAGE(xor_); - STAGE(colorburn); - STAGE(colordodge); - STAGE(darken); - STAGE(difference); - STAGE(exclusion); - STAGE(hardlight); - STAGE(lighten); - STAGE(overlay); - STAGE(softlight); - #undef STAGE - - } + void Init_hsw() { } } diff --git a/src/opts/SkOpts_sse41.cpp b/src/opts/SkOpts_sse41.cpp index d55978c8b7..3a37834c72 100644 --- a/src/opts/SkOpts_sse41.cpp +++ b/src/opts/SkOpts_sse41.cpp @@ -21,60 +21,58 @@ namespace SkOpts { srcover_srgb_srgb = sse41::srcover_srgb_srgb; blit_row_s32a_opaque = sse41::blit_row_s32a_opaque; - #define STAGE(stage) \ - body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage; \ - tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage##_tail + #define STAGE(stage, kCallNext) \ + stages_4 [SkRasterPipeline::stage] = stage_4 ; \ + stages_1_3[SkRasterPipeline::stage] = stage_1_3 - STAGE(store_565); - STAGE(store_srgb); - STAGE(store_f16); + STAGE(store_565 , false); + STAGE(store_srgb, false); + STAGE(store_f16 , false); - STAGE(load_s_565); - STAGE(load_s_srgb); - STAGE(load_s_f16); + STAGE(load_s_565 , true); + STAGE(load_s_srgb, true); + STAGE(load_s_f16 , true); - STAGE(load_d_565); - STAGE(load_d_srgb); - STAGE(load_d_f16); + STAGE(load_d_565 , true); + STAGE(load_d_srgb, true); + STAGE(load_d_f16 , true); - STAGE(scale_u8); + STAGE(scale_u8, true); - STAGE(lerp_u8); - STAGE(lerp_565); - #undef STAGE + STAGE(lerp_u8 , true); + STAGE(lerp_565 , true); + STAGE(lerp_constant_float, true); - #define STAGE(stage) \ - body[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage; \ - tail[SkRasterPipeline::stage] = (SkOpts::VoidFn)SK_OPTS_NS::stage + STAGE(constant_color, true); - // The commented-out stages don't actually benefit from SSE 4.1. - // To cut down on code bloat we skip them here, using the identical SSE2 defaults. + #undef STAGE - //STAGE(lerp_constant_float); - //STAGE(constant_color); + #define STAGE(stage) \ + stages_4 [SkRasterPipeline::stage] = SK_OPTS_NS::stage; \ + stages_1_3[SkRasterPipeline::stage] = SK_OPTS_NS::stage - //STAGE(dst); - //STAGE(dstatop); - //STAGE(dstin); - //STAGE(dstout); - //STAGE(dstover); - //STAGE(srcatop); - //STAGE(srcin); - //STAGE(srcout); - //STAGE(srcover); - //STAGE(clear); - //STAGE(modulate); - //STAGE(multiply); - //STAGE(plus_); - //STAGE(screen); - //STAGE(xor_); + STAGE(dst); + STAGE(dstatop); + STAGE(dstin); + STAGE(dstout); + STAGE(dstover); + STAGE(srcatop); + STAGE(srcin); + STAGE(srcout); + STAGE(srcover); + STAGE(clear); + STAGE(modulate); + STAGE(multiply); + STAGE(plus_); + STAGE(screen); + STAGE(xor_); STAGE(colorburn); STAGE(colordodge); - //STAGE(darken); - //STAGE(difference); - //STAGE(exclusion); + STAGE(darken); + STAGE(difference); + STAGE(exclusion); STAGE(hardlight); - //STAGE(lighten); + STAGE(lighten); STAGE(overlay); STAGE(softlight); #undef STAGE diff --git a/src/opts/SkRasterPipeline_opts.h b/src/opts/SkRasterPipeline_opts.h index 1d8b04452d..b0e6e1d2f9 100644 --- a/src/opts/SkRasterPipeline_opts.h +++ b/src/opts/SkRasterPipeline_opts.h @@ -13,147 +13,129 @@ #include "SkRasterPipeline.h" #include "SkSRGB.h" -using SkNf = SkRasterPipeline::V; -static constexpr auto N = sizeof(SkNf) / sizeof(float); -using SkNi = SkNx; -using SkNh = SkNx; - -#define SI static inline - -#define STAGE(name, kCallNext) \ - static SK_ALWAYS_INLINE void name##_kernel(void* ctx, size_t x, size_t tail, \ - SkNf& r, SkNf& g, SkNf& b, SkNf& a, \ - SkNf& dr, SkNf& dg, SkNf& db, SkNf& da); \ - SI void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail, \ - SkNf r, SkNf g, SkNf b, SkNf a, \ - SkNf dr, SkNf dg, SkNf db, SkNf da) { \ - name##_kernel(st->ctx(), x,0, r,g,b,a, dr,dg,db,da); \ - if (kCallNext) { \ - st->next(x,tail, r,g,b,a, dr,dg,db,da); \ - } \ - } \ - SI void SK_VECTORCALL name##_tail(SkRasterPipeline::Stage* st, size_t x, size_t tail, \ - SkNf r, SkNf g, SkNf b, SkNf a, \ - SkNf dr, SkNf dg, SkNf db, SkNf da) { \ - name##_kernel(st->ctx(), x,tail, r,g,b,a, dr,dg,db,da); \ - if (kCallNext) { \ - st->next(x,tail, r,g,b,a, dr,dg,db,da); \ - } \ - } \ - static SK_ALWAYS_INLINE void name##_kernel(void* ctx, size_t x, size_t tail, \ - SkNf& r, SkNf& g, SkNf& b, SkNf& a, \ - SkNf& dr, SkNf& dg, SkNf& db, SkNf& da) +using Kernel_Sk4f = void(void*, size_t, size_t, Sk4f&, Sk4f&, Sk4f&, Sk4f&, + Sk4f&, Sk4f&, Sk4f&, Sk4f&); + +// These are always static, and we _really_ want them to inline. +// If you find yourself wanting a non-inline stage, write a SkRasterPipeline::Fn directly. +#define KERNEL_Sk4f(name) \ + static SK_ALWAYS_INLINE void name(void* ctx, size_t x, size_t tail, \ + Sk4f& r, Sk4f& g, Sk4f& b, Sk4f& a, \ + Sk4f& dr, Sk4f& dg, Sk4f& db, Sk4f& da) + + +template +static inline void SK_VECTORCALL stage_4(SkRasterPipeline::Stage* st, size_t x, size_t tail, + Sk4f r, Sk4f g, Sk4f b, Sk4f a, + Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) { + // Passing 0 lets the optimizer completely drop any "if (tail) {...}" code in kernel. + kernel(st->ctx(), x,0, r,g,b,a, dr,dg,db,da); + if (kCallNext) { + st->next(x,tail, r,g,b,a, dr,dg,db,da); // It's faster to pass t here than 0. + } +} +template +static inline void SK_VECTORCALL stage_1_3(SkRasterPipeline::Stage* st, size_t x, size_t tail, + Sk4f r, Sk4f g, Sk4f b, Sk4f a, + Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) { +#if defined(__clang__) + __builtin_assume(tail > 0); // This flourish lets Clang compile away any tail==0 code. +#endif + kernel(st->ctx(), x,tail, r,g,b,a, dr,dg,db,da); + if (kCallNext) { + st->next(x,tail, r,g,b,a, dr,dg,db,da); + } +} // Many xfermodes apply the same logic to each channel. -#define RGBA_XFERMODE(name) \ - static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa, \ - const SkNf& d, const SkNf& da); \ - SI void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail, \ - SkNf r, SkNf g, SkNf b, SkNf a, \ - SkNf dr, SkNf dg, SkNf db, SkNf da) { \ - r = name##_kernel(r,a,dr,da); \ - g = name##_kernel(g,a,dg,da); \ - b = name##_kernel(b,a,db,da); \ - a = name##_kernel(a,a,da,da); \ - st->next(x,tail, r,g,b,a, dr,dg,db,da); \ - } \ - static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa, \ - const SkNf& d, const SkNf& da) +#define RGBA_XFERMODE_Sk4f(name) \ + static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa, \ + const Sk4f& d, const Sk4f& da); \ + static void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail, \ + Sk4f r, Sk4f g, Sk4f b, Sk4f a, \ + Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) { \ + r = name##_kernel(r,a,dr,da); \ + g = name##_kernel(g,a,dg,da); \ + b = name##_kernel(b,a,db,da); \ + a = name##_kernel(a,a,da,da); \ + st->next(x,tail, r,g,b,a, dr,dg,db,da); \ + } \ + static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa, \ + const Sk4f& d, const Sk4f& da) // Most of the rest apply the same logic to color channels and use srcover's alpha logic. -#define RGB_XFERMODE(name) \ - static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa, \ - const SkNf& d, const SkNf& da); \ - SI void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail, \ - SkNf r, SkNf g, SkNf b, SkNf a, \ - SkNf dr, SkNf dg, SkNf db, SkNf da) { \ - r = name##_kernel(r,a,dr,da); \ - g = name##_kernel(g,a,dg,da); \ - b = name##_kernel(b,a,db,da); \ - a = a + (da * (1.0f-a)); \ - st->next(x,tail, r,g,b,a, dr,dg,db,da); \ - } \ - static SK_ALWAYS_INLINE SkNf name##_kernel(const SkNf& s, const SkNf& sa, \ - const SkNf& d, const SkNf& da) - +#define RGB_XFERMODE_Sk4f(name) \ + static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa, \ + const Sk4f& d, const Sk4f& da); \ + static void SK_VECTORCALL name(SkRasterPipeline::Stage* st, size_t x, size_t tail, \ + Sk4f r, Sk4f g, Sk4f b, Sk4f a, \ + Sk4f dr, Sk4f dg, Sk4f db, Sk4f da) { \ + r = name##_kernel(r,a,dr,da); \ + g = name##_kernel(g,a,dg,da); \ + b = name##_kernel(b,a,db,da); \ + a = a + (da * (1.0f-a)); \ + st->next(x,tail, r,g,b,a, dr,dg,db,da); \ + } \ + static SK_ALWAYS_INLINE Sk4f name##_kernel(const Sk4f& s, const Sk4f& sa, \ + const Sk4f& d, const Sk4f& da) namespace SK_OPTS_NS { - SI void run_pipeline(size_t x, size_t n, - void (*vBodyStart)(), SkRasterPipeline::Stage* body, - void (*vTailStart)(), SkRasterPipeline::Stage* tail) { - auto bodyStart = (SkRasterPipeline::Fn)vBodyStart, - tailStart = (SkRasterPipeline::Fn)vTailStart; - SkNf v; // Fastest to start uninitialized. - while (n >= N) { - bodyStart(body, x,0, v,v,v,v, v,v,v,v); - x += N; - n -= N; - } - if (n > 0) { - tailStart(tail, x,n, v,v,v,v, v,v,v,v); - } - } - // Clamp colors into [0,1] premul (e.g. just before storing back to memory). - SI void clamp_01_premul(SkNf& r, SkNf& g, SkNf& b, SkNf& a) { - a = SkNf::Max(a, 0.0f); - r = SkNf::Max(r, 0.0f); - g = SkNf::Max(g, 0.0f); - b = SkNf::Max(b, 0.0f); - - a = SkNf::Min(a, 1.0f); - r = SkNf::Min(r, a); - g = SkNf::Min(g, a); - b = SkNf::Min(b, a); + static void clamp_01_premul(Sk4f& r, Sk4f& g, Sk4f& b, Sk4f& a) { + a = Sk4f::Max(a, 0.0f); + r = Sk4f::Max(r, 0.0f); + g = Sk4f::Max(g, 0.0f); + b = Sk4f::Max(b, 0.0f); + + a = Sk4f::Min(a, 1.0f); + r = Sk4f::Min(r, a); + g = Sk4f::Min(g, a); + b = Sk4f::Min(b, a); } - SI SkNf inv(const SkNf& x) { return 1.0f - x; } + static Sk4f inv(const Sk4f& x) { return 1.0f - x; } - SI SkNf lerp(const SkNf& from, const SkNf& to, const SkNf& cov) { - return SkNx_fma(to-from, cov, from); + static Sk4f lerp(const Sk4f& from, const Sk4f& to, const Sk4f& cov) { + return from + (to-from)*cov; } template - SI SkNx load_tail(size_t tail, const T* src) { - // TODO: better tail, maskload for 32- and 64-bit T - T buf[N] = {0}; + static SkNx<4,T> load_tail(size_t tail, const T* src) { if (tail) { - memcpy(buf, src, tail*sizeof(T)); - src = buf; + return SkNx<4,T>(src[0], (tail>1 ? src[1] : 0), (tail>2 ? src[2] : 0), 0); } - return SkNx::Load(src); + return SkNx<4,T>::Load(src); } template - SI void store_tail(size_t tail, const SkNx& v, T* dst) { - // TODO: better tail, maskstore for 32- and 64-bit T - T buf[N] = {0}; - v.store(tail ? buf : dst); - if (tail) { - memcpy(dst, buf, tail*sizeof(T)); + static void store_tail(size_t tail, const SkNx<4,T>& v, T* dst) { + switch(tail) { + case 0: return v.store(dst); + case 3: dst[2] = v[2]; + case 2: dst[1] = v[1]; + case 1: dst[0] = v[0]; } } - SI void from_565(const SkNh& _565, SkNf* r, SkNf* g, SkNf* b) { - auto _32_bit = SkNx_cast(_565); + static void from_565(const Sk4h& _565, Sk4f* r, Sk4f* g, Sk4f* b) { + Sk4i _32_bit = SkNx_cast(_565); *r = SkNx_cast(_32_bit & SK_R16_MASK_IN_PLACE) * (1.0f / SK_R16_MASK_IN_PLACE); *g = SkNx_cast(_32_bit & SK_G16_MASK_IN_PLACE) * (1.0f / SK_G16_MASK_IN_PLACE); *b = SkNx_cast(_32_bit & SK_B16_MASK_IN_PLACE) * (1.0f / SK_B16_MASK_IN_PLACE); } - SI SkNh to_565(const SkNf& r, const SkNf& g, const SkNf& b) { - return SkNx_cast( SkNx_cast(r * SK_R16_MASK + 0.5f) << SK_R16_SHIFT - | SkNx_cast(g * SK_G16_MASK + 0.5f) << SK_G16_SHIFT - | SkNx_cast(b * SK_B16_MASK + 0.5f) << SK_B16_SHIFT); + static Sk4h to_565(const Sk4f& r, const Sk4f& g, const Sk4f& b) { + return SkNx_cast( Sk4f_round(r * SK_R16_MASK) << SK_R16_SHIFT + | Sk4f_round(g * SK_G16_MASK) << SK_G16_SHIFT + | Sk4f_round(b * SK_B16_MASK) << SK_B16_SHIFT); } - STAGE(just_return, false) { } // The default shader produces a constant color (from the SkPaint). - STAGE(constant_color, true) { + KERNEL_Sk4f(constant_color) { auto color = (const SkPM4f*)ctx; r = color->r(); g = color->g(); @@ -162,8 +144,8 @@ namespace SK_OPTS_NS { } // s' = d(1-c) + sc, for a constant c. - STAGE(lerp_constant_float, true) { - SkNf c = *(const float*)ctx; + KERNEL_Sk4f(lerp_constant_float) { + Sk4f c = *(const float*)ctx; r = lerp(dr, r, c); g = lerp(dg, g, c); @@ -172,10 +154,10 @@ namespace SK_OPTS_NS { } // s' = sc for 8-bit c. - STAGE(scale_u8, true) { + KERNEL_Sk4f(scale_u8) { auto ptr = (const uint8_t*)ctx + x; - SkNf c = SkNx_cast(load_tail(tail, ptr)) * (1/255.0f); + Sk4f c = SkNx_cast(load_tail(tail, ptr)) * (1/255.0f); r = r*c; g = g*c; b = b*c; @@ -183,10 +165,10 @@ namespace SK_OPTS_NS { } // s' = d(1-c) + sc for 8-bit c. - STAGE(lerp_u8, true) { + KERNEL_Sk4f(lerp_u8) { auto ptr = (const uint8_t*)ctx + x; - SkNf c = SkNx_cast(load_tail(tail, ptr)) * (1/255.0f); + Sk4f c = SkNx_cast(load_tail(tail, ptr)) * (1/255.0f); r = lerp(dr, r, c); g = lerp(dg, g, c); b = lerp(db, b, c); @@ -194,9 +176,9 @@ namespace SK_OPTS_NS { } // s' = d(1-c) + sc for 565 c. - STAGE(lerp_565, true) { + KERNEL_Sk4f(lerp_565) { auto ptr = (const uint16_t*)ctx + x; - SkNf cr, cg, cb; + Sk4f cr, cg, cb; from_565(load_tail(tail, ptr), &cr, &cg, &cb); r = lerp(dr, r, cr); @@ -205,145 +187,155 @@ namespace SK_OPTS_NS { a = 1.0f; } - STAGE(load_d_565, true) { + KERNEL_Sk4f(load_d_565) { auto ptr = (const uint16_t*)ctx + x; from_565(load_tail(tail, ptr), &dr,&dg,&db); da = 1.0f; } - STAGE(load_s_565, true) { + KERNEL_Sk4f(load_s_565) { auto ptr = (const uint16_t*)ctx + x; from_565(load_tail(tail, ptr), &r,&g,&b); a = 1.0f; } - STAGE(store_565, false) { + KERNEL_Sk4f(store_565) { clamp_01_premul(r,g,b,a); auto ptr = (uint16_t*)ctx + x; store_tail(tail, to_565(r,g,b), ptr); } - STAGE(load_d_f16, true) { + KERNEL_Sk4f(load_d_f16) { auto ptr = (const uint64_t*)ctx + x; - uint64_t buf[N] = {0}; if (tail) { - memcpy(buf, ptr, tail*sizeof(uint64_t)); - ptr = buf; + auto p0 = SkHalfToFloat_finite_ftz(ptr[0]) , + p1 = tail>1 ? SkHalfToFloat_finite_ftz(ptr[1]) : Sk4f{0}, + p2 = tail>2 ? SkHalfToFloat_finite_ftz(ptr[2]) : Sk4f{0}; + dr = { p0[0],p1[0],p2[0],0 }; + dg = { p0[1],p1[1],p2[1],0 }; + db = { p0[2],p1[2],p2[2],0 }; + da = { p0[3],p1[3],p2[3],0 }; + return; } - SkNh rh, gh, bh, ah; - SkNh::Load4(ptr, &rh, &gh, &bh, &ah); + Sk4h rh, gh, bh, ah; + Sk4h::Load4(ptr, &rh, &gh, &bh, &ah); dr = SkHalfToFloat_finite_ftz(rh); dg = SkHalfToFloat_finite_ftz(gh); db = SkHalfToFloat_finite_ftz(bh); da = SkHalfToFloat_finite_ftz(ah); } - STAGE(load_s_f16, true) { + KERNEL_Sk4f(load_s_f16) { auto ptr = (const uint64_t*)ctx + x; - uint64_t buf[N] = {0}; if (tail) { - memcpy(buf, ptr, tail*sizeof(uint64_t)); - ptr = buf; + auto p0 = SkHalfToFloat_finite_ftz(ptr[0]) , + p1 = tail>1 ? SkHalfToFloat_finite_ftz(ptr[1]) : Sk4f{0}, + p2 = tail>2 ? SkHalfToFloat_finite_ftz(ptr[2]) : Sk4f{0}; + r = { p0[0],p1[0],p2[0],0 }; + g = { p0[1],p1[1],p2[1],0 }; + b = { p0[2],p1[2],p2[2],0 }; + a = { p0[3],p1[3],p2[3],0 }; + return; } - SkNh rh, gh, bh, ah; - SkNh::Load4(ptr, &rh, &gh, &bh, &ah); + Sk4h rh, gh, bh, ah; + Sk4h::Load4(ptr, &rh, &gh, &bh, &ah); r = SkHalfToFloat_finite_ftz(rh); g = SkHalfToFloat_finite_ftz(gh); b = SkHalfToFloat_finite_ftz(bh); a = SkHalfToFloat_finite_ftz(ah); } - STAGE(store_f16, false) { + KERNEL_Sk4f(store_f16) { clamp_01_premul(r,g,b,a); auto ptr = (uint64_t*)ctx + x; - uint64_t buf[N] = {0}; - SkNh::Store4(tail ? buf : ptr, SkFloatToHalf_finite_ftz(r), - SkFloatToHalf_finite_ftz(g), - SkFloatToHalf_finite_ftz(b), - SkFloatToHalf_finite_ftz(a)); - if (tail) { - memcpy(ptr, buf, tail*sizeof(uint64_t)); + switch (tail) { + case 0: return Sk4h::Store4(ptr, SkFloatToHalf_finite_ftz(r), + SkFloatToHalf_finite_ftz(g), + SkFloatToHalf_finite_ftz(b), + SkFloatToHalf_finite_ftz(a)); + + case 3: SkFloatToHalf_finite_ftz({r[2], g[2], b[2], a[2]}).store(ptr+2); + case 2: SkFloatToHalf_finite_ftz({r[1], g[1], b[1], a[1]}).store(ptr+1); + case 1: SkFloatToHalf_finite_ftz({r[0], g[0], b[0], a[0]}).store(ptr+0); } } // Load 8-bit SkPMColor-order sRGB. - STAGE(load_d_srgb, true) { + KERNEL_Sk4f(load_d_srgb) { auto ptr = (const uint32_t*)ctx + x; - auto px = load_tail(tail, ptr); - auto to_int = [](const SkNx& v) { return SkNi::Load(&v); }; - dr = sk_linear_from_srgb_math(to_int((px >> SK_R32_SHIFT) & 0xff)); - dg = sk_linear_from_srgb_math(to_int((px >> SK_G32_SHIFT) & 0xff)); - db = sk_linear_from_srgb_math(to_int((px >> SK_B32_SHIFT) & 0xff)); - da = (1/255.0f)*SkNx_cast(to_int( px >> SK_A32_SHIFT )); + auto px = load_tail(tail, (const int*)ptr); + dr = sk_linear_from_srgb_math((px >> SK_R32_SHIFT) & 0xff); + dg = sk_linear_from_srgb_math((px >> SK_G32_SHIFT) & 0xff); + db = sk_linear_from_srgb_math((px >> SK_B32_SHIFT) & 0xff); + da = (1/255.0f)*SkNx_cast((px >> SK_A32_SHIFT) & 0xff); } - STAGE(load_s_srgb, true) { + KERNEL_Sk4f(load_s_srgb) { auto ptr = (const uint32_t*)ctx + x; - auto px = load_tail(tail, ptr); - auto to_int = [](const SkNx& v) { return SkNi::Load(&v); }; - r = sk_linear_from_srgb_math(to_int((px >> SK_R32_SHIFT) & 0xff)); - g = sk_linear_from_srgb_math(to_int((px >> SK_G32_SHIFT) & 0xff)); - b = sk_linear_from_srgb_math(to_int((px >> SK_B32_SHIFT) & 0xff)); - a = (1/255.0f)*SkNx_cast(to_int( px >> SK_A32_SHIFT )); + auto px = load_tail(tail, (const int*)ptr); + r = sk_linear_from_srgb_math((px >> SK_R32_SHIFT) & 0xff); + g = sk_linear_from_srgb_math((px >> SK_G32_SHIFT) & 0xff); + b = sk_linear_from_srgb_math((px >> SK_B32_SHIFT) & 0xff); + a = (1/255.0f)*SkNx_cast((px >> SK_A32_SHIFT) & 0xff); } - STAGE(store_srgb, false) { + KERNEL_Sk4f(store_srgb) { clamp_01_premul(r,g,b,a); auto ptr = (uint32_t*)ctx + x; - store_tail(tail, ( sk_linear_to_srgb_noclamp(r) << SK_R32_SHIFT - | sk_linear_to_srgb_noclamp(g) << SK_G32_SHIFT - | sk_linear_to_srgb_noclamp(b) << SK_B32_SHIFT - | SkNx_cast(255.0f * a + 0.5f) << SK_A32_SHIFT ), (int*)ptr); + store_tail(tail, ( sk_linear_to_srgb_noclamp(r) << SK_R32_SHIFT + | sk_linear_to_srgb_noclamp(g) << SK_G32_SHIFT + | sk_linear_to_srgb_noclamp(b) << SK_B32_SHIFT + | Sk4f_round(255.0f * a) << SK_A32_SHIFT), (int*)ptr); } - RGBA_XFERMODE(clear) { return 0.0f; } - //RGBA_XFERMODE(src) { return s; } // This would be a no-op stage, so we just omit it. - RGBA_XFERMODE(dst) { return d; } - - RGBA_XFERMODE(srcatop) { return s*da + d*inv(sa); } - RGBA_XFERMODE(srcin) { return s * da; } - RGBA_XFERMODE(srcout) { return s * inv(da); } - RGBA_XFERMODE(srcover) { return SkNx_fma(d, inv(sa), s); } - RGBA_XFERMODE(dstatop) { return srcatop_kernel(d,da,s,sa); } - RGBA_XFERMODE(dstin) { return srcin_kernel (d,da,s,sa); } - RGBA_XFERMODE(dstout) { return srcout_kernel (d,da,s,sa); } - RGBA_XFERMODE(dstover) { return srcover_kernel(d,da,s,sa); } - - RGBA_XFERMODE(modulate) { return s*d; } - RGBA_XFERMODE(multiply) { return s*inv(da) + d*inv(sa) + s*d; } - RGBA_XFERMODE(plus_) { return s + d; } - RGBA_XFERMODE(screen) { return s + d - s*d; } - RGBA_XFERMODE(xor_) { return s*inv(da) + d*inv(sa); } - - RGB_XFERMODE(colorburn) { + RGBA_XFERMODE_Sk4f(clear) { return 0.0f; } + //RGBA_XFERMODE_Sk4f(src) { return s; } // This would be a no-op stage, so we just omit it. + RGBA_XFERMODE_Sk4f(dst) { return d; } + + RGBA_XFERMODE_Sk4f(srcatop) { return s*da + d*inv(sa); } + RGBA_XFERMODE_Sk4f(srcin) { return s * da; } + RGBA_XFERMODE_Sk4f(srcout) { return s * inv(da); } + RGBA_XFERMODE_Sk4f(srcover) { return s + inv(sa)*d; } + RGBA_XFERMODE_Sk4f(dstatop) { return srcatop_kernel(d,da,s,sa); } + RGBA_XFERMODE_Sk4f(dstin) { return srcin_kernel (d,da,s,sa); } + RGBA_XFERMODE_Sk4f(dstout) { return srcout_kernel (d,da,s,sa); } + RGBA_XFERMODE_Sk4f(dstover) { return srcover_kernel(d,da,s,sa); } + + RGBA_XFERMODE_Sk4f(modulate) { return s*d; } + RGBA_XFERMODE_Sk4f(multiply) { return s*inv(da) + d*inv(sa) + s*d; } + RGBA_XFERMODE_Sk4f(plus_) { return s + d; } + RGBA_XFERMODE_Sk4f(screen) { return s + d - s*d; } + RGBA_XFERMODE_Sk4f(xor_) { return s*inv(da) + d*inv(sa); } + + RGB_XFERMODE_Sk4f(colorburn) { return (d == da ).thenElse(d + s*inv(da), (s == 0.0f).thenElse(s + d*inv(sa), - sa*(da - SkNf::Min(da, (da-d)*sa/s)) + s*inv(da) + d*inv(sa))); + sa*(da - Sk4f::Min(da, (da-d)*sa/s)) + s*inv(da) + d*inv(sa))); } - RGB_XFERMODE(colordodge) { + RGB_XFERMODE_Sk4f(colordodge) { return (d == 0.0f).thenElse(d + s*inv(da), (s == sa ).thenElse(s + d*inv(sa), - sa*SkNf::Min(da, (d*sa)/(sa - s)) + s*inv(da) + d*inv(sa))); + sa*Sk4f::Min(da, (d*sa)/(sa - s)) + s*inv(da) + d*inv(sa))); } - RGB_XFERMODE(darken) { return s + d - SkNf::Max(s*da, d*sa); } - RGB_XFERMODE(difference) { return s + d - 2.0f*SkNf::Min(s*da,d*sa); } - RGB_XFERMODE(exclusion) { return s + d - 2.0f*s*d; } - RGB_XFERMODE(hardlight) { + RGB_XFERMODE_Sk4f(darken) { return s + d - Sk4f::Max(s*da, d*sa); } + RGB_XFERMODE_Sk4f(difference) { return s + d - 2.0f*Sk4f::Min(s*da,d*sa); } + RGB_XFERMODE_Sk4f(exclusion) { return s + d - 2.0f*s*d; } + RGB_XFERMODE_Sk4f(hardlight) { return s*inv(da) + d*inv(sa) + (2.0f*s <= sa).thenElse(2.0f*s*d, sa*da - 2.0f*(da-d)*(sa-s)); } - RGB_XFERMODE(lighten) { return s + d - SkNf::Min(s*da, d*sa); } - RGB_XFERMODE(overlay) { return hardlight_kernel(d,da,s,sa); } - RGB_XFERMODE(softlight) { - SkNf m = (da > 0.0f).thenElse(d / da, 0.0f), + RGB_XFERMODE_Sk4f(lighten) { return s + d - Sk4f::Min(s*da, d*sa); } + RGB_XFERMODE_Sk4f(overlay) { return hardlight_kernel(d,da,s,sa); } + RGB_XFERMODE_Sk4f(softlight) { + Sk4f m = (da > 0.0f).thenElse(d / da, 0.0f), s2 = 2.0f*s, m4 = 4.0f*m; @@ -351,7 +343,7 @@ namespace SK_OPTS_NS { // 1. dark src? // 2. light src, dark dst? // 3. light src, light dst? - SkNf darkSrc = d*(sa + (s2 - sa)*(1.0f - m)), // Used in case 1. + Sk4f darkSrc = d*(sa + (s2 - sa)*(1.0f - m)), // Used in case 1. darkDst = (m4*m4 + m4)*(m - 1.0f) + 7.0f*m, // Used in case 2. liteDst = m.rsqrt().invert() - m, // Used in case 3. liteSrc = d*sa + da*(s2 - sa) * (4.0f*d <= da).thenElse(darkDst, liteDst); // 2 or 3? @@ -359,9 +351,8 @@ namespace SK_OPTS_NS { } } -#undef SI -#undef STAGE -#undef RGBA_XFERMODE -#undef RGB_XFERMODE +#undef KERNEL_Sk4f +#undef RGB_XFERMODE_Sk4f +#undef RGB_XFERMODE_Sk4f #endif//SkRasterPipeline_opts_DEFINED -- cgit v1.2.3