Avoid SkFixed overflow in decal bitmap procs

The check for decal mode can overflow in SkFixed.  Promote to
64bit (48.16) instead.

Also update can_truncate_to_fixed_for_decal() to take SkFixed params and
used it in ClampX_ClampY_filter_scale_SSE2().

BUG=chromium:675444
R=reed@google.com

CQ_INCLUDE_TRYBOTS=skia.primary:Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-SKNX_NO_SIMD

Change-Id: I759464cdaa5c005159e38e32167fb1937e2a1d28
Reviewed-on: https://skia-review.googlesource.com/6538
Reviewed-by: Cary Clark <caryclark@google.com>
Commit-Queue: Florin Malita <fmalita@chromium.org>

1 files changed, 23 insertions, 7 deletions

diff --git a/src/core/SkBitmapProcState_utils.h b/src/core/SkBitmapProcState_utils.h
index 3c4c1fa8c6..4609ff34e7 100644
--- a/src/core/SkBitmapProcState_utils.h
+++ b/src/core/SkBitmapProcState_utils.h
@@ -1,10 +1,17 @@
+/*
+ * Copyright 2017 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
 #ifndef SkBitmapProcState_utils_DEFINED
 #define SkBitmapProcState_utils_DEFINED
 
 // Helper to ensure that when we shift down, we do it w/o sign-extension
 // so the caller doesn't have to manually mask off the top 16 bits
 //
-static unsigned SK_USHIFT16(unsigned x) {
+static inline unsigned SK_USHIFT16(unsigned x) {
     return x >> 16;
 }
 
@@ -18,10 +25,10 @@ static unsigned SK_USHIFT16(unsigned x) {
  *  the decal_ function just operates on SkFixed. If that were changed, we could
  *  skip the very_small test here.
  */
-static inline bool can_truncate_to_fixed_for_decal(SkFractionalInt frX,
-                                                   SkFractionalInt frDx,
+static inline bool can_truncate_to_fixed_for_decal(SkFixed fx,
+                                                   SkFixed dx,
                                                    int count, unsigned max) {
-    SkFixed dx = SkFractionalIntToFixed(frDx);
+    SkASSERT(count > 0);
 
     // if decal_ kept SkFractionalInt precision, this would just be dx <= 0
     // I just made up the 1/256. Just don't want to perceive accumulated error
@@ -30,11 +37,20 @@ static inline bool can_truncate_to_fixed_for_decal(SkFractionalInt frX,
         return false;
     }
 
+    // Note: it seems the test should be (fx <= max && lastFx <= max); but
+    // historically it's been a strict inequality check, and changing produces
+    // unexpected diffs.  Further investigation is needed.
+
     // We cast to unsigned so we don't have to check for negative values, which
     // will now appear as very large positive values, and thus fail our test!
-    SkFixed fx = SkFractionalIntToFixed(frX);
-    return (unsigned)SkFixedFloorToInt(fx) <= max &&
-           (unsigned)SkFixedFloorToInt(fx + dx * (count - 1)) < max;
+    if ((unsigned)SkFixedFloorToInt(fx) >= max) {
+        return false;
+    }
+
+    // Promote to 64bit (48.16) to avoid overflow.
+    const uint64_t lastFx = fx + sk_64_mul(dx, count - 1);
+
+    return sk_64_isS32(lastFx) && (unsigned)SkFixedFloorToInt(sk_64_asS32(lastFx)) < max;
 }
 
 #endif /* #ifndef SkBitmapProcState_utils_DEFINED */