/* NEON optimized code (C) COPYRIGHT 2009 Motorola * * Use of this source code is governed by a BSD-style license that can be * found in the LICENSE file. */ /* * Modifications done in-house at Motorola * * this is a clone of SkBitmapProcState_matrix.h * and has been tuned to work with the NEON unit. * * Still going back and forth between whether this approach * (clone the entire SkBitmapProcState_matrix.h file or * if I should put just the modified routines in here and * then use a construct like #define DONT_DO_THIS_FUNCTION or * something like that... * * This is for the ClampX_ClampY instance * */ #include /* * This has been modified on the knowledge that (at the time) * we had the following macro definitions in the parent file * * #define MAKENAME(suffix) ClampX_ClampY ## suffix * #define TILEX_PROCF(fx, max) SkClampMax((fx) >> 16, max) * #define TILEY_PROCF(fy, max) SkClampMax((fy) >> 16, max) * #define TILEX_LOW_BITS(fx, max) (((fx) >> 12) & 0xF) * #define TILEY_LOW_BITS(fy, max) (((fy) >> 12) & 0xF) * #define CHECK_FOR_DECAL */ /* SkClampMax(val,max) -- bound to 0..max */ #define SCALE_NOFILTER_NAME MAKENAME(_nofilter_scale) #define SCALE_FILTER_NAME MAKENAME(_filter_scale) #define AFFINE_NOFILTER_NAME MAKENAME(_nofilter_affine) #define AFFINE_FILTER_NAME MAKENAME(_filter_affine) #define PERSP_NOFILTER_NAME MAKENAME(_nofilter_persp) #define PERSP_FILTER_NAME MAKENAME(_filter_persp) #define PACK_FILTER_X_NAME MAKENAME(_pack_filter_x) #define PACK_FILTER_Y_NAME MAKENAME(_pack_filter_y) #ifndef PREAMBLE #define PREAMBLE(state) #define PREAMBLE_PARAM_X #define PREAMBLE_PARAM_Y #define PREAMBLE_ARG_X #define PREAMBLE_ARG_Y #endif static void SCALE_NOFILTER_NAME(const SkBitmapProcState& s, uint32_t xy[], int count, int x, int y) { SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | SkMatrix::kScale_Mask)) == 0); PREAMBLE(s); // we store y, x, x, x, x, x const unsigned maxX = s.fBitmap->width() - 1; SkFixed fx; { SkPoint pt; s.fInvProc(s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, SkIntToScalar(y) + SK_ScalarHalf, &pt); fx = SkScalarToFixed(pt.fY); const unsigned maxY = s.fBitmap->height() - 1; *xy++ = TILEY_PROCF(fx, maxY); fx = SkScalarToFixed(pt.fX); } if (0 == maxX) { // all of the following X values must be 0 memset(xy, 0, count * sizeof(uint16_t)); return; } const SkFixed dx = s.fInvSx; #ifdef CHECK_FOR_DECAL // test if we don't need to apply the tile proc if ((unsigned)(fx >> 16) <= maxX && (unsigned)((fx + dx * (count - 1)) >> 16) <= maxX) { decal_nofilter_scale_neon(xy, fx, dx, count); return; } #endif int i; /* very much like done in decal_nofilter, but with * an extra clamping function applied. * TILEX_PROCF(fx,max) SkClampMax((fx)>>16, max) */ if (count >= 8) { /* SkFixed is 16.16 fixed point */ SkFixed dx2 = dx+dx; SkFixed dx4 = dx2+dx2; SkFixed dx8 = dx4+dx4; /* now build fx/fx+dx/fx+2dx/fx+3dx */ SkFixed fx1, fx2, fx3; int32x4_t lbase, hbase; int16_t *dst16 = (int16_t *)xy; fx1 = fx+dx; fx2 = fx1+dx; fx3 = fx2+dx; /* build my template(s) */ /* avoid the 'lbase unitialized' warning */ lbase = vdupq_n_s32(fx); lbase = vsetq_lane_s32(fx1, lbase, 1); lbase = vsetq_lane_s32(fx2, lbase, 2); lbase = vsetq_lane_s32(fx3, lbase, 3); hbase = vaddq_s32(lbase, vdupq_n_s32(dx4)); /* store & bump */ do { int32x4_t lout; int32x4_t hout; int16x8_t hi16; /* get the hi 16s of all those 32s */ lout = lbase; hout = hbase; /* this sets up all lout's then all hout's in hout */ asm ("vuzpq.16 %q0, %q1" : "+w" (lout), "+w" (hout)); hi16 = vreinterpretq_s16_s32(hout); /* clamp & output */ hi16 = vmaxq_s16(hi16, vdupq_n_s16(0)); hi16 = vminq_s16(hi16, vdupq_n_s16(maxX)); vst1q_s16(dst16, hi16); /* but preserving base & on to the next */ lbase = vaddq_s32 (lbase, vdupq_n_s32(dx8)); hbase = vaddq_s32 (hbase, vdupq_n_s32(dx8)); dst16 += 8; count -= 8; fx += dx8; } while (count >= 8); xy = (uint32_t *) dst16; } uint16_t* xx = (uint16_t*)xy; for (i = count; i > 0; --i) { *xx++ = TILEX_PROCF(fx, maxX); fx += dx; } } // note: we could special-case on a matrix which is skewed in X but not Y. // this would require a more general setup thatn SCALE does, but could use // SCALE's inner loop that only looks at dx static void AFFINE_NOFILTER_NAME(const SkBitmapProcState& s, uint32_t xy[], int count, int x, int y) { SkASSERT(s.fInvType & SkMatrix::kAffine_Mask); SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | SkMatrix::kScale_Mask | SkMatrix::kAffine_Mask)) == 0); PREAMBLE(s); SkPoint srcPt; s.fInvProc(s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, SkIntToScalar(y) + SK_ScalarHalf, &srcPt); SkFixed fx = SkScalarToFixed(srcPt.fX); SkFixed fy = SkScalarToFixed(srcPt.fY); SkFixed dx = s.fInvSx; SkFixed dy = s.fInvKy; int maxX = s.fBitmap->width() - 1; int maxY = s.fBitmap->height() - 1; /* NEON lets us do an 8x unrolling */ if (count >= 8) { /* SkFixed is 16.16 fixed point */ SkFixed dx4 = dx * 4; SkFixed dy4 = dy * 4; SkFixed dx8 = dx * 8; SkFixed dy8 = dy * 8; int32x4_t xbase, ybase; int32x4_t x2base, y2base; int16_t *dst16 = (int16_t *) xy; /* my sets of maxx/maxy for clamping */ int32_t maxpair = (maxX&0xffff) | ((maxY&0xffff)<<16); int16x8_t maxXY = vreinterpretq_s16_s32(vdupq_n_s32(maxpair)); /* now build fx/fx+dx/fx+2dx/fx+3dx */ /* avoid the 'xbase unitialized' warning...*/ xbase = vdupq_n_s32(fx); xbase = vsetq_lane_s32(fx+dx, xbase, 1); xbase = vsetq_lane_s32(fx+dx+dx, xbase, 2); xbase = vsetq_lane_s32(fx+dx+dx+dx, xbase, 3); /* same for fy */ /* avoid the 'ybase unitialized' warning...*/ ybase = vdupq_n_s32(fy); ybase = vsetq_lane_s32(fy+dy, ybase, 1); ybase = vsetq_lane_s32(fy+dy+dy, ybase, 2); ybase = vsetq_lane_s32(fy+dy+dy+dy, ybase, 3); x2base = vaddq_s32(xbase, vdupq_n_s32(dx4)); y2base = vaddq_s32(ybase, vdupq_n_s32(dy4)); /* store & bump */ do { int32x4_t xout, yout; int32x4_t x2out, y2out; int16x8_t hi16, hi16_2; xout = xbase; yout = ybase; /* overlay y's low16 with hi16 from x */ /* so we properly shifted xyxyxyxy */ yout = vsriq_n_s32(yout, xout, 16); hi16 = vreinterpretq_s16_s32 (yout); /* do the clamping; both guys get 0's */ hi16 = vmaxq_s16 (hi16, vdupq_n_s16(0)); hi16 = vminq_s16 (hi16, maxXY); vst1q_s16 (dst16, hi16); /* and for the other 4 pieces of this iteration */ x2out = x2base; y2out = y2base; /* overlay y's low16 with hi16 from x */ /* so we properly shifted xyxyxyxy */ y2out = vsriq_n_s32(y2out, x2out, 16); hi16_2 = vreinterpretq_s16_s32 (y2out); /* do the clamping; both guys get 0's */ hi16_2 = vmaxq_s16 (hi16_2, vdupq_n_s16(0)); hi16_2 = vminq_s16 (hi16_2, maxXY); /* RBE: gcc regenerates dst16+8 all the time instead * of folding it into an addressing mode. *sigh* */ vst1q_s16 (dst16+8, hi16_2); /* moving base and on to the next */ xbase = vaddq_s32 (xbase, vdupq_n_s32 (dx8)); ybase = vaddq_s32 (ybase, vdupq_n_s32 (dy8)); x2base = vaddq_s32 (x2base, vdupq_n_s32 (dx8)); y2base = vaddq_s32 (y2base, vdupq_n_s32 (dy8)); dst16 += 16; /* 8x32 aka 16x16 */ count -= 8; fx += dx8; fy += dy8; } while (count >= 8); xy = (uint32_t *) dst16; } for (int i = count; i > 0; --i) { *xy++ = (TILEY_PROCF(fy, maxY) << 16) | TILEX_PROCF(fx, maxX); fx += dx; fy += dy; } } #undef DEBUG_PERSP_NOFILTER static void PERSP_NOFILTER_NAME(const SkBitmapProcState& s, uint32_t* SK_RESTRICT xy, int count, int x, int y) { SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask); PREAMBLE(s); /* max{X,Y} are int here, but later shown/assumed to fit in 16 bits */ int maxX = s.fBitmap->width() - 1; int maxY = s.fBitmap->height() - 1; SkPerspIter iter(s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, SkIntToScalar(y) + SK_ScalarHalf, count); while ((count = iter.next()) != 0) { const SkFixed* SK_RESTRICT srcXY = iter.getXY(); #if defined(DEBUG_PERSP_NOFILTER) /* debugging stuff */ const SkFixed *end_srcXY = srcXY + (count*2); uint32_t *end_xy = xy + (count); const SkFixed *base_srcXY = srcXY; uint32_t *base_xy = xy; int base_count = count; #endif #if 1 // 2009/9/30: crashes in ApiDemos - Views - Animation - 3D Transition // 2009/10/9: reworked to avoid illegal (but allowed by gas) insn /* srcXY is a batch of 32 bit numbers X0,Y0,X1,Y1... * but we immediately discard the low 16 bits... * so what we're going to do is vld4, which will give us * xlo,xhi,ylo,yhi distribution and we can ignore the 'lo' * parts.... */ if (count >= 8) { int16_t *mysrc = (int16_t *) srcXY; int16_t *mydst = (int16_t *) xy; int16x4_t maxX4 = vdup_n_s16((int16_t)maxX); int16x4_t maxY4 = vdup_n_s16((int16_t)maxY); int16x4_t zero4 = vdup_n_s16(0); /* The constructs with local blocks for register assignments * and asm() instructions is to make keep any hard register * assignments to as small a scope as possible. and to avoid * burning call-preserved hard registers on the vld/vst * instructions. */ do { int16x4_t xhi, yhi; int16x4_t x2hi, y2hi; /* vld4 does the de-interleaving for us */ { register int16x4_t t_xlo asm("d0"); register int16x4_t t_xhi asm("d1"); register int16x4_t t_ylo asm("d2"); register int16x4_t t_yhi asm("d3"); asm ("vld4.16 {d0-d3},[%4] /* xlo=%P0 xhi=%P1 ylo=%P2 yhi=%P3 */" : "=w" (t_xlo), "=w" (t_xhi), "=w" (t_ylo), "=w" (t_yhi) : "r" (mysrc) ); xhi = t_xhi; yhi = t_yhi; } /* clamp X>>16 (aka xhi) to 0..maxX */ xhi = vmax_s16(xhi, zero4); /* now 0.. */ xhi = vmin_s16(xhi, maxX4); /* now 0..maxX */ /* clamp Y>>16 (aka yhi) to 0..maxY */ yhi = vmax_s16(yhi, zero4); /* now 0.. */ yhi = vmin_s16(yhi, maxY4); /* now 0..maxY */ /* deal with the second set of numbers */ { register int16x4_t t_xlo asm("d4"); register int16x4_t t_xhi asm("d5"); register int16x4_t t_ylo asm("d6"); register int16x4_t t_yhi asm("d7"); /* offset == 256 bits == 32 bytes == 8 longs == 16 shorts */ asm ("vld4.16 {d4-d7},[%4] /* xlo=%P0 xhi=%P1 ylo=%P2 yhi=%P3 */" : "=w" (t_xlo), "=w" (t_xhi), "=w" (t_ylo), "=w" (t_yhi) : "r" (mysrc+16) ); x2hi = t_xhi; y2hi = t_yhi; } /* clamp the second 4 here */ if (0) { extern void rbe(void); rbe(); } /* clamp X>>16 (aka xhi) to 0..maxX */ x2hi = vmax_s16(x2hi, zero4); /* now 0.. */ x2hi = vmin_s16(x2hi, maxX4); /* now 0..maxX */ /* clamp Y>>16 (aka yhi) to 0..maxY */ y2hi = vmax_s16(y2hi, zero4); /* now 0.. */ y2hi = vmin_s16(y2hi, maxY4); /* now 0..maxY */ /* we're storing as {x,y}s: x is [0], y is [1] */ /* we'll use vst2 to make this happen */ { register int16x4_t out_x asm("d16") = xhi; register int16x4_t out_y asm("d17") = yhi; asm ("vst2.16 {d16-d17},[%2] /* xlo=%P0 xhi=%P1 */" : : "w" (out_x), "w" (out_y), "r" (mydst) ); } { register int16x4_t out_x asm("d18") = x2hi; register int16x4_t out_y asm("d19") = y2hi; asm ("vst2.16 {d18-d19},[%2] /* xlo=%P0 xhi=%P1 */" : : "w" (out_x), "w" (out_y), "r" (mydst+8) ); } /* XXX: gcc isn't interleaving these with the NEON ops * but i think that all the scoreboarding works out */ count -= 8; /* 8 iterations */ mysrc += 32; /* 16 longs, aka 32 shorts */ mydst += 16; /* 16 shorts, aka 8 longs */ } while (count >= 8); /* get xy and srcXY fixed up */ srcXY = (const SkFixed *) mysrc; xy = (uint32_t *) mydst; } #endif while (--count >= 0) { *xy++ = (TILEY_PROCF(srcXY[1], maxY) << 16) | TILEX_PROCF(srcXY[0], maxX); srcXY += 2; } #if defined(DEBUG_PERSP_NOFILTER) /* for checking our NEON-produced results against vanilla code */ { int bad = (-1); for (int i = 0; i < base_count; i++) { uint32_t val; val = (TILEY_PROCF (base_srcXY[i * 2 + 1], maxY) << 16) | TILEX_PROCF (base_srcXY[i * 2 + 0], maxX); if (val != base_xy[i]) { bad = i; break; } } if (bad >= 0) { SkDebugf("clamp-nofilter-persp failed piece %d\n", bad); SkDebugf(" maxX %08x maxY %08x\n", maxX, maxY); bad -= (bad & 0x7); /* align */ for (int i = bad; i < bad + 8; i++) { uint32_t val; val = (TILEY_PROCF (base_srcXY[i * 2 + 1], maxY) << 16) | TILEX_PROCF (base_srcXY[i * 2 + 0], maxX); SkDebugf("%d: got %08x want %08x srcXY[0] %08x srcXY[1] %08x\n", i, base_xy[i], val, base_srcXY[i * 2 + 0], base_srcXY[i * 2 + 1]); } SkDebugf ("---\n"); } if (end_xy != xy) { SkDebugf("xy ended at %08x, should be %08x\n", xy, end_xy); } if (end_srcXY != srcXY) { SkDebugf("srcXY ended at %08x, should be %08x\n", srcXY, end_srcXY); } } #endif } } #undef DEBUG_PERSP_NOFILTER ////////////////////////////////////////////////////////////////////////////// static inline uint32_t PACK_FILTER_Y_NAME(SkFixed f, unsigned max, SkFixed one PREAMBLE_PARAM_Y) { unsigned i = TILEY_PROCF(f, max); i = (i << 4) | TILEY_LOW_BITS(f, max); return (i << 14) | (TILEY_PROCF((f + one), max)); } static inline uint32_t PACK_FILTER_X_NAME(SkFixed f, unsigned max, SkFixed one PREAMBLE_PARAM_X) { unsigned i = TILEX_PROCF(f, max); i = (i << 4) | TILEX_LOW_BITS(f, max); return (i << 14) | (TILEX_PROCF((f + one), max)); } static void SCALE_FILTER_NAME(const SkBitmapProcState& s, uint32_t xy[], int count, int x, int y) { SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | SkMatrix::kScale_Mask)) == 0); SkASSERT(s.fInvKy == 0); PREAMBLE(s); const unsigned maxX = s.fBitmap->width() - 1; const SkFixed one = s.fFilterOneX; const SkFixed dx = s.fInvSx; SkFixed fx; { SkPoint pt; s.fInvProc(s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, SkIntToScalar(y) + SK_ScalarHalf, &pt); const SkFixed fy = SkScalarToFixed(pt.fY) - (s.fFilterOneY >> 1); const unsigned maxY = s.fBitmap->height() - 1; // compute our two Y values up front *xy++ = PACK_FILTER_Y_NAME(fy, maxY, s.fFilterOneY PREAMBLE_ARG_Y); // now initialize fx fx = SkScalarToFixed(pt.fX) - (one >> 1); } #ifdef CHECK_FOR_DECAL // test if we don't need to apply the tile proc if (dx > 0 && (unsigned)(fx >> 16) <= maxX && (unsigned)((fx + dx * (count - 1)) >> 16) < maxX) { decal_filter_scale_neon(xy, fx, dx, count); } else #endif if (count >= 4) { int32x4_t wide_one, wide_fx, wide_fx1, wide_i, wide_lo; #if 0 /* verification hooks -- see below */ SkFixed debug_fx = fx; int count_done = 0; #endif wide_fx = vdupq_n_s32(fx); wide_fx = vsetq_lane_s32(fx+dx, wide_fx, 1); wide_fx = vsetq_lane_s32(fx+dx+dx, wide_fx, 2); wide_fx = vsetq_lane_s32(fx+dx+dx+dx, wide_fx, 3); wide_one = vdupq_n_s32(one); while (count >= 4) { /* original expands to: * unsigned i = SkClampMax((f) >> 16, max); * i = (i << 4) | (((f) >> 12) & 0xF); * return (i << 14) | (SkClampMax(((f + one)) >> 16, max)); */ /* i = SkClampMax(f>>16, maxX) */ wide_i = vmaxq_s32(vshrq_n_s32(wide_fx,16), vdupq_n_s32(0)); wide_i = vminq_s32(wide_i, vdupq_n_s32(maxX)); /* i<<4 | TILEX_LOW_BITS(fx) */ wide_lo = vshrq_n_s32(wide_fx, 12); wide_i = vsliq_n_s32(wide_lo, wide_i, 4); /* i<<14 */ wide_i = vshlq_n_s32(wide_i, 14); /* SkClampMax(((f + one)) >> 16, max) */ wide_fx1 = vaddq_s32(wide_fx, wide_one); wide_fx1 = vmaxq_s32(vshrq_n_s32(wide_fx1,16), vdupq_n_s32(0)); wide_fx1 = vminq_s32(wide_fx1, vdupq_n_s32(maxX)); /* final combination */ wide_i = vorrq_s32(wide_i, wide_fx1); vst1q_u32(xy, vreinterpretq_u32_s32(wide_i)); #if 0 /* having a verification hook is a good idea */ /* use debug_fx, debug_fx+dx, etc. */ for (int i=0;i<4;i++) { uint32_t want = PACK_FILTER_X_NAME(debug_fx, maxX, one PREAMBLE_ARG_X); if (xy[i] != want) { /* print a nastygram */ SkDebugf("clamp-filter-scale fails\n"); SkDebugf("got %08x want %08x\n", xy[i], want); SkDebugf("fx %08x debug_fx %08x dx %08x done %d\n", fx, debug_fx, dx, count_done); SkDebugf(" maxX %08x one %08x\n", maxX, one); } debug_fx += dx; count_done++; } #endif wide_fx += vdupq_n_s32(dx+dx+dx+dx); fx += dx+dx+dx+dx; xy += 4; count -= 4; } } while (--count >= 0) { *xy++ = PACK_FILTER_X_NAME(fx, maxX, one PREAMBLE_ARG_X); fx += dx; } } static void AFFINE_FILTER_NAME(const SkBitmapProcState& s, uint32_t xy[], int count, int x, int y) { SkASSERT(s.fInvType & SkMatrix::kAffine_Mask); SkASSERT((s.fInvType & ~(SkMatrix::kTranslate_Mask | SkMatrix::kScale_Mask | SkMatrix::kAffine_Mask)) == 0); PREAMBLE(s); SkPoint srcPt; s.fInvProc(s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, SkIntToScalar(y) + SK_ScalarHalf, &srcPt); SkFixed oneX = s.fFilterOneX; SkFixed oneY = s.fFilterOneY; SkFixed fx = SkScalarToFixed(srcPt.fX) - (oneX >> 1); SkFixed fy = SkScalarToFixed(srcPt.fY) - (oneY >> 1); SkFixed dx = s.fInvSx; SkFixed dy = s.fInvKy; unsigned maxX = s.fBitmap->width() - 1; unsigned maxY = s.fBitmap->height() - 1; if (count >= 4) { int32x4_t wide_i, wide_lo; int32x4_t wide_fx, wide_onex, wide_fx1; int32x4_t wide_fy, wide_oney, wide_fy1; #undef AFFINE_DEBUG #if defined(AFFINE_DEBUG) SkFixed fyp = fy; SkFixed fxp = fx; uint32_t *xyp = xy; int count_done = 0; #endif wide_fx = vdupq_n_s32(fx); wide_fx = vsetq_lane_s32(fx+dx, wide_fx, 1); wide_fx = vsetq_lane_s32(fx+dx+dx, wide_fx, 2); wide_fx = vsetq_lane_s32(fx+dx+dx+dx, wide_fx, 3); wide_fy = vdupq_n_s32(fy); wide_fy = vsetq_lane_s32(fy+dy, wide_fy, 1); wide_fy = vsetq_lane_s32(fy+dy+dy, wide_fy, 2); wide_fy = vsetq_lane_s32(fy+dy+dy+dy, wide_fy, 3); wide_onex = vdupq_n_s32(oneX); wide_oney = vdupq_n_s32(oneY); while (count >= 4) { int32x4_t wide_x; int32x4_t wide_y; /* do the X side, then the Y side, then interleave them */ /* original expands to: * unsigned i = SkClampMax((f) >> 16, max); * i = (i << 4) | (((f) >> 12) & 0xF); * return (i << 14) | (SkClampMax(((f + one)) >> 16, max)); */ /* i = SkClampMax(f>>16, maxX) */ wide_i = vmaxq_s32(vshrq_n_s32(wide_fx,16), vdupq_n_s32(0)); wide_i = vminq_s32(wide_i, vdupq_n_s32(maxX)); /* i<<4 | TILEX_LOW_BITS(fx) */ wide_lo = vshrq_n_s32(wide_fx, 12); wide_i = vsliq_n_s32(wide_lo, wide_i, 4); /* i<<14 */ wide_i = vshlq_n_s32(wide_i, 14); /* SkClampMax(((f + one)) >> 16, max) */ wide_fx1 = vaddq_s32(wide_fx, wide_onex); wide_fx1 = vmaxq_s32(vshrq_n_s32(wide_fx1,16), vdupq_n_s32(0)); wide_fx1 = vminq_s32(wide_fx1, vdupq_n_s32(maxX)); /* final combination */ wide_x = vorrq_s32(wide_i, wide_fx1); /* And now the Y side */ /* i = SkClampMax(f>>16, maxX) */ wide_i = vmaxq_s32(vshrq_n_s32(wide_fy,16), vdupq_n_s32(0)); wide_i = vminq_s32(wide_i, vdupq_n_s32(maxY)); /* i<<4 | TILEX_LOW_BITS(fx) */ wide_lo = vshrq_n_s32(wide_fy, 12); wide_i = vsliq_n_s32(wide_lo, wide_i, 4); /* i<<14 */ wide_i = vshlq_n_s32(wide_i, 14); /* SkClampMax(((f + one)) >> 16, max) */ wide_fy1 = vaddq_s32(wide_fy, wide_oney); wide_fy1 = vmaxq_s32(vshrq_n_s32(wide_fy1,16), vdupq_n_s32(0)); wide_fy1 = vminq_s32(wide_fy1, vdupq_n_s32(maxY)); /* final combination */ wide_y = vorrq_s32(wide_i, wide_fy1); /* interleave as YXYXYXYX as part of the storing */ { /* vst2.32 needs side-by-side registers */ register int32x4_t t_x asm("q1"); register int32x4_t t_y asm("q0"); t_x = wide_x; t_y = wide_y; asm ("vst2.32 {q0-q1},[%2] /* y=%q0 x=%q1 */" : : "w" (t_y), "w" (t_x), "r" (xy) ); } #if defined(AFFINE_DEBUG) /* make sure we're good here -- check the 4 we just output */ for (int i = 0; i<4;i++) { uint32_t val; val = PACK_FILTER_Y_NAME(fyp, maxY, oneY PREAMBLE_ARG_Y); if (val != xy[i*2+0]) { /* print a nastygram */ SkDebugf("clamp-filter-affine fails\n"); SkDebugf("[bad-y] got %08x want %08x\n", xy[i*2+0], val); SkDebugf("fy %08x fxp %08x fyp %08x dx %08x dy %08x done %d\n", fy, fxp, fyp, dx, dy, count_done); SkDebugf(" maxY %08x oneY %08x\n", maxY, oneY); } val = PACK_FILTER_X_NAME(fxp, maxX, oneX PREAMBLE_ARG_X); if (val != xy[i*2+1]) { /* print a nastygram */ SkDebugf("clamp-filter-affine fails\n"); SkDebugf("[bad-x] got %08x want %08x\n", xy[i*2+1], val); SkDebugf("fx %08x fxp %08x fyp %08x dx %08x dy %08x done %d\n", fx, fxp, fyp, dx, dy, count_done); SkDebugf(" maxX %08x one %08x\n", maxX, oneX); } fyp += dy; fxp += dx; count_done++; } #endif wide_fx += vdupq_n_s32(dx+dx+dx+dx); fx += dx+dx+dx+dx; wide_fy += vdupq_n_s32(dy+dy+dy+dy); fy += dy+dy+dy+dy; xy += 8; /* 4 x's, 4 y's */ count -= 4; } } while (--count >= 0) { /* NB: writing Y/X */ *xy++ = PACK_FILTER_Y_NAME(fy, maxY, oneY PREAMBLE_ARG_Y); fy += dy; *xy++ = PACK_FILTER_X_NAME(fx, maxX, oneX PREAMBLE_ARG_X); fx += dx; } } static void PERSP_FILTER_NAME(const SkBitmapProcState& s, uint32_t* SK_RESTRICT xy, int count, int x, int y) { SkASSERT(s.fInvType & SkMatrix::kPerspective_Mask); PREAMBLE(s); unsigned maxX = s.fBitmap->width() - 1; unsigned maxY = s.fBitmap->height() - 1; SkFixed oneX = s.fFilterOneX; SkFixed oneY = s.fFilterOneY; SkPerspIter iter(s.fInvMatrix, SkIntToScalar(x) + SK_ScalarHalf, SkIntToScalar(y) + SK_ScalarHalf, count); while ((count = iter.next()) != 0) { const SkFixed* SK_RESTRICT srcXY = iter.getXY(); if (count >= 4) { int32x4_t wide_i, wide_lo; int32x4_t wide_fx1; int32x4_t wide_fy1; int32x4_t wide_x, wide_y; while (count >= 4) { /* RBE: it's good, but: * -- we spill a constant that could be easily regnerated * [perhaps tweak gcc's NEON constant costs?] */ /* load src: x-y-x-y-x-y-x-y */ { register int32x4_t q0 asm ("q0"); register int32x4_t q1 asm ("q1"); asm ("vld2.32 {q0-q1},[%2] /* x=%q0 y=%q1 */" : "=w" (q0), "=w" (q1) : "r" (srcXY)); wide_x = q0; wide_y = q1; } /* do the X side, then the Y side, then interleave them */ wide_x = vsubq_s32(wide_x, vdupq_n_s32 (oneX>>1)); /* original expands to: * unsigned i = SkClampMax((f) >> 16, max); * i = (i << 4) | (((f) >> 12) & 0xF); * return (i << 14) | (SkClampMax(((f + one)) >> 16, max)); */ /* i = SkClampMax(f>>16, maxX) */ wide_i = vmaxq_s32 (vshrq_n_s32 (wide_x, 16), vdupq_n_s32 (0)); wide_i = vminq_s32 (wide_i, vdupq_n_s32 (maxX)); /* i<<4 | TILEX_LOW_BITS(fx) */ wide_lo = vshrq_n_s32 (wide_x, 12); wide_i = vsliq_n_s32 (wide_lo, wide_i, 4); /* i<<14 */ wide_i = vshlq_n_s32 (wide_i, 14); /* SkClampMax(((f + one)) >> 16, max) */ wide_fx1 = vaddq_s32 (wide_x, vdupq_n_s32(oneX)); wide_fx1 = vmaxq_s32 (vshrq_n_s32 (wide_fx1, 16), vdupq_n_s32 (0)); wide_fx1 = vminq_s32 (wide_fx1, vdupq_n_s32 (maxX)); /* final combination */ wide_x = vorrq_s32 (wide_i, wide_fx1); /* And now the Y side */ wide_y = vsubq_s32(wide_y, vdupq_n_s32 (oneY>>1)); /* i = SkClampMax(f>>16, maxX) */ wide_i = vmaxq_s32 (vshrq_n_s32 (wide_y, 16), vdupq_n_s32 (0)); wide_i = vminq_s32 (wide_i, vdupq_n_s32 (maxY)); /* i<<4 | TILEX_LOW_BITS(fx) */ wide_lo = vshrq_n_s32 (wide_y, 12); wide_i = vsliq_n_s32 (wide_lo, wide_i, 4); /* i<<14 */ wide_i = vshlq_n_s32 (wide_i, 14); /* SkClampMax(((f + one)) >> 16, max) */ /* wide_fy1_1 and wide_fy1_2 are just temporary variables to * work-around an ICE in debug */ int32x4_t wide_fy1_1 = vaddq_s32 (wide_y, vdupq_n_s32(oneY)); int32x4_t wide_fy1_2 = vmaxq_s32 (vshrq_n_s32 (wide_fy1_1, 16), vdupq_n_s32 (0)); wide_fy1 = vminq_s32 (wide_fy1_2, vdupq_n_s32 (maxY)); /* final combination */ wide_y = vorrq_s32 (wide_i, wide_fy1); /* switch them around; have to do it this way to get them * in the proper registers to match our instruction */ /* iteration bookkeeping, ahead of the asm() for scheduling */ srcXY += 2*4; count -= 4; /* store interleaved as y-x-y-x-y-x-y-x (NB != read order) */ { register int32x4_t q0 asm ("q0") = wide_y; register int32x4_t q1 asm ("q1") = wide_x; asm ("vst2.32 {q0-q1},[%2] /* y=%q0 x=%q1 */" : : "w" (q0), "w" (q1), "r" (xy)); } /* on to the next iteration */ /* count, srcXY are handled above */ xy += 2*4; } } /* was do-while; NEON code invalidates original count>0 assumption */ while (--count >= 0) { /* NB: we read x/y, we write y/x */ *xy++ = PACK_FILTER_Y_NAME(srcXY[1] - (oneY >> 1), maxY, oneY PREAMBLE_ARG_Y); *xy++ = PACK_FILTER_X_NAME(srcXY[0] - (oneX >> 1), maxX, oneX PREAMBLE_ARG_X); srcXY += 2; } } } const SkBitmapProcState::MatrixProc MAKENAME(_Procs)[] = { SCALE_NOFILTER_NAME, SCALE_FILTER_NAME, AFFINE_NOFILTER_NAME, AFFINE_FILTER_NAME, PERSP_NOFILTER_NAME, PERSP_FILTER_NAME }; #undef MAKENAME #undef TILEX_PROCF #undef TILEY_PROCF #ifdef CHECK_FOR_DECAL #undef CHECK_FOR_DECAL #endif #undef SCALE_NOFILTER_NAME #undef SCALE_FILTER_NAME #undef AFFINE_NOFILTER_NAME #undef AFFINE_FILTER_NAME #undef PERSP_NOFILTER_NAME #undef PERSP_FILTER_NAME #undef PREAMBLE #undef PREAMBLE_PARAM_X #undef PREAMBLE_PARAM_Y #undef PREAMBLE_ARG_X #undef PREAMBLE_ARG_Y #undef TILEX_LOW_BITS #undef TILEY_LOW_BITS