Threaded generation of software paths

All information needed by the thread is captured by the prepare
callback object, the lambda captures a pointer to that, and does the
mask render. Once it's done, it signals the semaphore (also owned by the
callback). The callback defers the semaphore wait even longer (into the
ASAP upload), so the odds of waiting for the thread are REALLY low.

Also did a bunch of cleanup along the way, and put in some trace markers
so we can monitor how well this is working.

Traces of a GM that includes GPU and SW path rendering (path-reverse):

Original:
    https://screenshot.googleplex.com/f5BG3901tQg.png
Threaded, with wait in the callback (notice pre flush callback blocking):
    https://screenshot.googleplex.com/htOSZFE2s04.png
Current version, with wait deferred to ASAP upload function:
    https://screenshot.googleplex.com/GHjD0U3C34q.png

Bug: skia:
Change-Id: I3d5a230bbd68eb35e1f0574b308485c691435790
Reviewed-on: https://skia-review.googlesource.com/36560
Commit-Queue: Brian Osman <brianosman@google.com>
Reviewed-by: Brian Salomon <bsalomon@google.com>

18 files changed, 296 insertions, 91 deletions

diff --git a/bench/nanobench.cpp b/bench/nanobench.cpp
index ab7c689309..ef3bdf776f 100644
--- a/bench/nanobench.cpp
+++ b/bench/nanobench.cpp
@@ -30,6 +30,7 @@
 #include "SkCodec.h"
 #include "SkCommonFlags.h"
 #include "SkCommonFlagsConfig.h"
+#include "SkCommonFlagsGpuThreads.h"
 #include "SkCommonFlagsPathRenderer.h"
 #include "SkData.h"
 #include "SkDebugfTracer.h"
@@ -1139,6 +1140,7 @@ int main(int argc, char** argv) {
 #if SK_SUPPORT_GPU
     GrContextOptions grContextOpts;
     grContextOpts.fGpuPathRenderers = CollectGpuPathRenderersFromFlags();
+    grContextOpts.fExecutor = GpuExecutorForTools();
     gGrFactory.reset(new GrContextFactory(grContextOpts));
 #endif
 
diff --git a/dm/DM.cpp b/dm/DM.cpp
index a442e09050..1e02d25b7b 100644
--- a/dm/DM.cpp
+++ b/dm/DM.cpp
@@ -18,6 +18,7 @@
 #include "SkColorSpacePriv.h"
 #include "SkCommonFlags.h"
 #include "SkCommonFlagsConfig.h"
+#include "SkCommonFlagsGpuThreads.h"
 #include "SkCommonFlagsPathRenderer.h"
 #include "SkData.h"
 #include "SkDocument.h"
@@ -858,10 +859,19 @@ static Sink* create_sink(const GrContextOptions& grCtxOptions, const SkCommandLi
                      "GM tests will be skipped.\n", gpuConfig->getTag().c_str());
                 return nullptr;
             }
-            return new GPUSink(contextType, contextOverrides, gpuConfig->getSamples(),
-                               gpuConfig->getUseDIText(), gpuConfig->getColorType(),
-                               gpuConfig->getAlphaType(), sk_ref_sp(gpuConfig->getColorSpace()),
-                               FLAGS_gpu_threading, grCtxOptions);
+            if (gpuConfig->getTestThreading()) {
+                return new GPUThreadTestingSink(contextType, contextOverrides,
+                                                gpuConfig->getSamples(), gpuConfig->getUseDIText(),
+                                                gpuConfig->getColorType(),
+                                                gpuConfig->getAlphaType(),
+                                                sk_ref_sp(gpuConfig->getColorSpace()),
+                                                FLAGS_gpu_threading, grCtxOptions);
+            } else {
+                return new GPUSink(contextType, contextOverrides, gpuConfig->getSamples(),
+                                   gpuConfig->getUseDIText(), gpuConfig->getColorType(),
+                                   gpuConfig->getAlphaType(), sk_ref_sp(gpuConfig->getColorSpace()),
+                                   FLAGS_gpu_threading, grCtxOptions);
+            }
         }
     }
 #endif
@@ -1320,6 +1330,7 @@ int main(int argc, char** argv) {
     GrContextOptions grCtxOptions;
 #if SK_SUPPORT_GPU
     grCtxOptions.fGpuPathRenderers = CollectGpuPathRenderersFromFlags();
+    grCtxOptions.fExecutor = GpuExecutorForTools();
 #endif
 
     JsonWriter::DumpJson();  // It's handy for the bots to assume this is ~never missing.
diff --git a/dm/DMSrcSink.cpp b/dm/DMSrcSink.cpp
index f9e5fd3d21..47b10e42bf 100644
--- a/dm/DMSrcSink.cpp
+++ b/dm/DMSrcSink.cpp
@@ -20,6 +20,7 @@
 #include "SkDebugCanvas.h"
 #include "SkDeferredCanvas.h"
 #include "SkDocument.h"
+#include "SkExecutor.h"
 #include "SkImageGenerator.h"
 #include "SkImageGeneratorCG.h"
 #include "SkImageGeneratorWIC.h"
@@ -61,6 +62,7 @@
 DEFINE_bool(multiPage, false, "For document-type backends, render the source"
             " into multiple pages");
 DEFINE_bool(RAW_threading, true, "Allow RAW decodes to run on multiple threads?");
+DECLARE_int32(gpuThreads);
 
 using sk_gpu_test::GrContextFactory;
 
@@ -1315,8 +1317,13 @@ GPUSink::GPUSink(GrContextFactory::ContextType ct,
 
 DEFINE_bool(drawOpClip, false, "Clip each GrDrawOp to its device bounds for testing.");
 
-Error GPUSink::draw(const Src& src, SkBitmap* dst, SkWStream*, SkString* log) const {
-    GrContextOptions grOptions = fBaseContextOptions;
+Error GPUSink::draw(const Src& src, SkBitmap* dst, SkWStream* dstStream, SkString* log) const {
+    return this->onDraw(src, dst, dstStream, log, fBaseContextOptions);
+}
+
+Error GPUSink::onDraw(const Src& src, SkBitmap* dst, SkWStream*, SkString* log,
+                      const GrContextOptions& baseOptions) const {
+    GrContextOptions grOptions = baseOptions;
 
     src.modifyGrContextOptions(&grOptions);
 
@@ -1368,6 +1375,58 @@ Error GPUSink::draw(const Src& src, SkBitmap* dst, SkWStream*, SkString* log) co
 
 /*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
 
+GPUThreadTestingSink::GPUThreadTestingSink(GrContextFactory::ContextType ct,
+                                           GrContextFactory::ContextOverrides overrides,
+                                           int samples,
+                                           bool diText,
+                                           SkColorType colorType,
+                                           SkAlphaType alphaType,
+                                           sk_sp<SkColorSpace> colorSpace,
+                                           bool threaded,
+                                           const GrContextOptions& grCtxOptions)
+        : INHERITED(ct, overrides, samples, diText, colorType, alphaType, std::move(colorSpace),
+                    threaded, grCtxOptions)
+        , fExecutor(SkExecutor::MakeThreadPool(FLAGS_gpuThreads)) {
+    SkASSERT(fExecutor);
+}
+
+Error GPUThreadTestingSink::draw(const Src& src, SkBitmap* dst, SkWStream* wStream,
+                                 SkString* log) const {
+    // Draw twice, once with worker threads, and once without. Verify that we get the same result.
+    // Also, force us to only use the software path renderer, so we really stress-test the threaded
+    // version of that code.
+    GrContextOptions contextOptions = this->baseContextOptions();
+    contextOptions.fGpuPathRenderers = GrContextOptions::GpuPathRenderers::kNone;
+
+    contextOptions.fExecutor = fExecutor.get();
+    Error err = this->onDraw(src, dst, wStream, log, contextOptions);
+    if (!err.isEmpty() || !dst) {
+        return err;
+    }
+
+    SkBitmap reference;
+    SkString refLog;
+    SkDynamicMemoryWStream refStream;
+    contextOptions.fExecutor = nullptr;
+    Error refErr = this->onDraw(src, &reference, &refStream, &refLog, contextOptions);
+    if (!refErr.isEmpty()) {
+        return refErr;
+    }
+
+    // The dimensions are a property of the Src only, and so should be identical.
+    SkASSERT(reference.getSize() == dst->getSize());
+    if (reference.getSize() != dst->getSize()) {
+        return "Dimensions don't match reference";
+    }
+    // All SkBitmaps in DM are tight, so this comparison is easy.
+    if (0 != memcmp(reference.getPixels(), dst->getPixels(), reference.getSize())) {
+        return "Pixels don't match reference";
+    }
+    return "";
+}
+
+/*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
+
 static Error draw_skdocument(const Src& src, SkDocument* doc, SkWStream* dst) {
     if (src.size().isEmpty()) {
         return "Source has empty dimensions";
@@ -1558,7 +1617,7 @@ static Error check_against_reference(const SkBitmap* bitmap, const Src& src, Sin
         if (reference.getSize() != bitmap->getSize()) {
             return "Dimensions don't match reference";
         }
-        // All SkBitmaps in DM are pre-locked and tight, so this comparison is easy.
+        // All SkBitmaps in DM are tight, so this comparison is easy.
         if (0 != memcmp(reference.getPixels(), bitmap->getPixels(), reference.getSize())) {
             return "Pixels don't match reference";
         }
diff --git a/dm/DMSrcSink.h b/dm/DMSrcSink.h
index 117514995a..8b6ee0e5f4 100644
--- a/dm/DMSrcSink.h
+++ b/dm/DMSrcSink.h
@@ -310,6 +310,9 @@ public:
             bool threaded, const GrContextOptions& grCtxOptions);
 
     Error draw(const Src&, SkBitmap*, SkWStream*, SkString*) const override;
+    Error onDraw(const Src&, SkBitmap*, SkWStream*, SkString*,
+                 const GrContextOptions& baseOptions) const;
+
     bool serial() const override { return !fThreaded; }
     const char* fileExtension() const override { return "png"; }
     SinkFlags flags() const override {
@@ -317,6 +320,8 @@ public:
                                                       : SinkFlags::kNotMultisampled;
         return SinkFlags{ SinkFlags::kGPU, SinkFlags::kDirect, ms };
     }
+    const GrContextOptions& baseContextOptions() const { return fBaseContextOptions; }
+
 private:
     sk_gpu_test::GrContextFactory::ContextType        fContextType;
     sk_gpu_test::GrContextFactory::ContextOverrides   fContextOverrides;
@@ -329,6 +334,22 @@ private:
     GrContextOptions                                  fBaseContextOptions;
 };
 
+class GPUThreadTestingSink : public GPUSink {
+public:
+    GPUThreadTestingSink(sk_gpu_test::GrContextFactory::ContextType,
+                         sk_gpu_test::GrContextFactory::ContextOverrides, int samples, bool diText,
+                         SkColorType colorType, SkAlphaType alphaType,
+                         sk_sp<SkColorSpace> colorSpace, bool threaded,
+                         const GrContextOptions& grCtxOptions);
+
+    Error draw(const Src&, SkBitmap*, SkWStream*, SkString*) const override;
+
+private:
+    std::unique_ptr<SkExecutor> fExecutor;
+
+    typedef GPUSink INHERITED;
+};
+
 class PDFSink : public Sink {
 public:
     PDFSink(bool pdfa, SkScalar rasterDpi) : fPDFA(pdfa), fRasterDpi(rasterDpi) {}
diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-TSAN.json b/infra/bots/recipes/test.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-TSAN.json
index 38bac0ddc1..19217f0129 100644
--- a/infra/bots/recipes/test.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-TSAN.json
+++ b/infra/bots/recipes/test.expected/Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Release-TSAN.json
@@ -178,6 +178,8 @@
       "image",
       "colorImage",
       "svg",
+      "--gpuThreads",
+      "8",
       "--blacklist",
       "~8888",
       "svg",
diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC5PPYH-GPU-IntelHD405-x86_64-Debug.json b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC5PPYH-GPU-IntelHD405-x86_64-Debug.json
index c00676ccc8..200f8852c1 100644
--- a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC5PPYH-GPU-IntelHD405-x86_64-Debug.json
+++ b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC5PPYH-GPU-IntelHD405-x86_64-Debug.json
@@ -260,6 +260,7 @@
       "glsrgb",
       "gles",
       "glesdft",
+      "gltestthreading",
       "--src",
       "tests",
       "gm",
diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Release.json b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Release.json
index c3b060f203..b62d851e7f 100644
--- a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Release.json
+++ b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUC6i5SYK-GPU-IntelIris540-x86_64-Release.json
@@ -260,6 +260,7 @@
       "glsrgb",
       "gles",
       "glesdft",
+      "gltestthreading",
       "--src",
       "tests",
       "gm",
diff --git a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUCDE3815TYKHE-GPU-IntelBayTrail-x86_64-Debug.json b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUCDE3815TYKHE-GPU-IntelBayTrail-x86_64-Debug.json
index f3b5141f82..1cbdb4e88a 100644
--- a/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUCDE3815TYKHE-GPU-IntelBayTrail-x86_64-Debug.json
+++ b/infra/bots/recipes/test.expected/Test-Ubuntu16-Clang-NUCDE3815TYKHE-GPU-IntelBayTrail-x86_64-Debug.json
@@ -260,6 +260,7 @@
       "glsrgb",
       "gles",
       "glesdft",
+      "gltestthreading",
       "--src",
       "tests",
       "gm",
diff --git a/infra/bots/recipes/test.py b/infra/bots/recipes/test.py
index 69ec34afe5..094475b7fb 100644
--- a/infra/bots/recipes/test.py
+++ b/infra/bots/recipes/test.py
@@ -148,8 +148,10 @@ def dm_flags(api, bot):
 
     # We want to test both the OpenGL config and the GLES config on Linux Intel:
     # GL is used by Chrome, GLES is used by ChromeOS.
+    # Also do the Ganesh threading verification test (render with and without
+    # worker threads, using only the SW path renderer, and compare the results).
     if 'Intel' in bot and api.vars.is_linux:
-      configs.extend(['gles', 'glesdft', 'glessrgb'])
+      configs.extend(['gles', 'glesdft', 'glessrgb', 'gltestthreading'])
 
     # The following devices do not support glessrgb.
     if 'glessrgb' in configs:
@@ -486,6 +488,7 @@ def dm_flags(api, bot):
     match.extend(['~Once', '~Shared'])  # Not sure what's up with these tests.
 
   if 'TSAN' in bot:
+    args.extend(['--gpuThreads', '8'])
     match.extend(['~ReadWriteAlpha'])   # Flaky on TSAN-covered on nvidia bots.
     match.extend(['~RGBA4444TextureTest',  # Flakier than they are important.
                   '~RGB565TextureTest'])
diff --git a/src/gpu/GrSWMaskHelper.cpp b/src/gpu/GrSWMaskHelper.cpp
index eb3986525e..af0d1bdc22 100644
--- a/src/gpu/GrSWMaskHelper.cpp
+++ b/src/gpu/GrSWMaskHelper.cpp
@@ -6,12 +6,12 @@
  */
 
 #include "GrSWMaskHelper.h"
+
 #include "GrContext.h"
 #include "GrContextPriv.h"
 #include "GrShape.h"
 #include "GrSurfaceContext.h"
 #include "GrTextureProxy.h"
-#include "SkDistanceFieldGen.h"
 
 /*
  * Convert a boolean operation into a transfer mode code
@@ -76,13 +76,13 @@ bool GrSWMaskHelper::init(const SkIRect& resultBounds, const SkMatrix* matrix) {
     SkIRect bounds = SkIRect::MakeWH(resultBounds.width(), resultBounds.height());
 
     const SkImageInfo bmImageInfo = SkImageInfo::MakeA8(bounds.width(), bounds.height());
-    if (!fPixels.tryAlloc(bmImageInfo)) {
+    if (!fPixels->tryAlloc(bmImageInfo)) {
         return false;
     }
-    fPixels.erase(0);
+    fPixels->erase(0);
 
     sk_bzero(&fDraw, sizeof(fDraw));
-    fDraw.fDst      = fPixels;
+    fDraw.fDst      = *fPixels;
     fRasterClip.setRect(bounds);
     fDraw.fRC       = &fRasterClip;
     fDraw.fMatrix   = &fMatrix;
@@ -92,8 +92,8 @@ bool GrSWMaskHelper::init(const SkIRect& resultBounds, const SkMatrix* matrix) {
 sk_sp<GrTextureProxy> GrSWMaskHelper::toTextureProxy(GrContext* context, SkBackingFit fit) {
     GrSurfaceDesc desc;
     desc.fOrigin = kTopLeft_GrSurfaceOrigin;
-    desc.fWidth = fPixels.width();
-    desc.fHeight = fPixels.height();
+    desc.fWidth = fPixels->width();
+    desc.fHeight = fPixels->height();
     desc.fConfig = kAlpha_8_GrPixelConfig;
 
     sk_sp<GrSurfaceContext> sContext = context->contextPriv().makeDeferredSurfaceContext(
@@ -105,39 +105,9 @@ sk_sp<GrTextureProxy> GrSWMaskHelper::toTextureProxy(GrContext* context, SkBacki
     }
 
     SkImageInfo ii = SkImageInfo::MakeA8(desc.fWidth, desc.fHeight);
-    if (!sContext->writePixels(ii, fPixels.addr(), fPixels.rowBytes(), 0, 0)) {
+    if (!sContext->writePixels(ii, fPixels->addr(), fPixels->rowBytes(), 0, 0)) {
         return nullptr;
     }
 
     return sContext->asTextureProxyRef();
 }
-
-/**
- * Convert mask generation results to a signed distance field
- */
-void GrSWMaskHelper::toSDF(unsigned char* sdf) {
-    SkGenerateDistanceFieldFromA8Image(sdf, (const unsigned char*)fPixels.addr(),
-                                       fPixels.width(), fPixels.height(), fPixels.rowBytes());
-}
-
-////////////////////////////////////////////////////////////////////////////////
-/**
- * Software rasterizes shape to A8 mask and uploads the result to a scratch texture. Returns the
- * resulting texture on success; nullptr on failure.
- */
-sk_sp<GrTextureProxy> GrSWMaskHelper::DrawShapeMaskToTexture(GrContext* context,
-                                                             const GrShape& shape,
-                                                             const SkIRect& resultBounds,
-                                                             GrAA aa,
-                                                             SkBackingFit fit,
-                                                             const SkMatrix* matrix) {
-    GrSWMaskHelper helper;
-
-    if (!helper.init(resultBounds, matrix)) {
-        return nullptr;
-    }
-
-    helper.drawShape(shape, SkRegion::kReplace_Op, aa, 0xFF);
-
-    return helper.toTextureProxy(context, fit);
-}
diff --git a/src/gpu/GrSWMaskHelper.h b/src/gpu/GrSWMaskHelper.h
index a98d58b991..4e1363c3df 100644
--- a/src/gpu/GrSWMaskHelper.h
+++ b/src/gpu/GrSWMaskHelper.h
@@ -8,22 +8,15 @@
 #ifndef GrSWMaskHelper_DEFINED
 #define GrSWMaskHelper_DEFINED
 
-#include "GrColor.h"
-#include "GrRenderTargetContext.h"
 #include "SkAutoPixmapStorage.h"
-#include "SkBitmap.h"
 #include "SkDraw.h"
 #include "SkMatrix.h"
 #include "SkRasterClip.h"
 #include "SkRegion.h"
 #include "SkTypes.h"
 
-class GrClip;
-class GrPaint;
 class GrShape;
-class GrStyle;
-class GrTexture;
-struct GrUserStencilSettings;
+class GrTextureProxy;
 
 /**
  * The GrSWMaskHelper helps generate clip masks using the software rendering
@@ -34,14 +27,15 @@ struct GrUserStencilSettings;
  *
  *      draw one or more paths/rects specifying the required boolean ops
  *
- *   toTexture();   // to get it from the internal bitmap to the GPU
+ *   toTextureProxy();   // to get it from the internal bitmap to the GPU
  *
  * The result of this process will be the final mask (on the GPU) in the
  * upper left hand corner of the texture.
  */
 class GrSWMaskHelper : SkNoncopyable {
 public:
-    GrSWMaskHelper() { }
+    GrSWMaskHelper(SkAutoPixmapStorage* pixels = nullptr)
+            : fPixels(pixels ? pixels : &fPixelsStorage) { }
 
     // set up the internal state in preparation for draws. Since many masks
     // may be accumulated in the helper during creation, "resultBounds"
@@ -57,28 +51,17 @@ public:
 
     sk_sp<GrTextureProxy> toTextureProxy(GrContext*, SkBackingFit fit);
 
-    // Convert mask generation results to a signed distance field
-    void toSDF(unsigned char* sdf);
-
     // Reset the internal bitmap
     void clear(uint8_t alpha) {
-        fPixels.erase(SkColorSetARGB(alpha, 0xFF, 0xFF, 0xFF));
+        fPixels->erase(SkColorSetARGB(alpha, 0xFF, 0xFF, 0xFF));
     }
 
-    // Canonical usage utility that draws a single path and uploads it
-    // to the GPU. The result is returned.
-    static sk_sp<GrTextureProxy> DrawShapeMaskToTexture(GrContext*,
-                                                        const GrShape&,
-                                                        const SkIRect& resultBounds,
-                                                        GrAA,
-                                                        SkBackingFit,
-                                                        const SkMatrix* matrix);
-
 private:
-    SkMatrix            fMatrix;
-    SkAutoPixmapStorage fPixels;
-    SkDraw              fDraw;
-    SkRasterClip        fRasterClip;
+    SkMatrix             fMatrix;
+    SkAutoPixmapStorage* fPixels;
+    SkAutoPixmapStorage  fPixelsStorage;
+    SkDraw               fDraw;
+    SkRasterClip         fRasterClip;
 
     typedef SkNoncopyable INHERITED;
 };
diff --git a/src/gpu/GrSoftwarePathRenderer.cpp b/src/gpu/GrSoftwarePathRenderer.cpp
index 27ba8e6436..e76af89933 100644
--- a/src/gpu/GrSoftwarePathRenderer.cpp
+++ b/src/gpu/GrSoftwarePathRenderer.cpp
@@ -8,9 +8,16 @@
 #include "GrSoftwarePathRenderer.h"
 #include "GrAuditTrail.h"
 #include "GrClip.h"
+#include "GrContextPriv.h"
 #include "GrGpuResourcePriv.h"
+#include "GrOpFlushState.h"
+#include "GrOpList.h"
 #include "GrResourceProvider.h"
 #include "GrSWMaskHelper.h"
+#include "SkMakeUnique.h"
+#include "SkSemaphore.h"
+#include "SkTaskGroup.h"
+#include "SkTraceEvent.h"
 #include "ops/GrDrawOp.h"
 #include "ops/GrRectOpFactory.h"
 
@@ -145,10 +152,84 @@ void GrSoftwarePathRenderer::DrawToTargetWithShapeMask(
     maskMatrix.preConcat(viewMatrix);
     paint.addCoverageFragmentProcessor(GrSimpleTextureEffect::Make(
                 std::move(proxy), nullptr, maskMatrix, GrSamplerParams::kNone_FilterMode));
-    renderTargetContext->addDrawOp(clip,
-                                   GrRectOpFactory::MakeNonAAFillWithLocalMatrix(
-                                           std::move(paint), SkMatrix::I(), invert, dstRect,
-                                           GrAAType::kNone, &userStencilSettings));
+    DrawNonAARect(renderTargetContext, std::move(paint), userStencilSettings, clip, SkMatrix::I(),
+                  dstRect, invert);
+}
+
+static sk_sp<GrTextureProxy> make_deferred_mask_texture_proxy(GrContext* context, SkBackingFit fit,
+                                                              int width, int height) {
+    GrSurfaceDesc desc;
+    desc.fOrigin = kTopLeft_GrSurfaceOrigin;
+    desc.fWidth = width;
+    desc.fHeight = height;
+    desc.fConfig = kAlpha_8_GrPixelConfig;
+
+    sk_sp<GrSurfaceContext> sContext =
+            context->contextPriv().makeDeferredSurfaceContext(desc, fit, SkBudgeted::kYes);
+    if (!sContext || !sContext->asTextureProxy()) {
+        return nullptr;
+    }
+    return sContext->asTextureProxyRef();
+}
+
+namespace {
+
+class GrMaskUploaderPrepareCallback : public GrPrepareCallback {
+public:
+    GrMaskUploaderPrepareCallback(sk_sp<GrTextureProxy> proxy, const SkIRect& maskBounds,
+                                  const SkMatrix& viewMatrix, const GrShape& shape, GrAA aa)
+            : fProxy(std::move(proxy))
+            , fMaskBounds(maskBounds)
+            , fViewMatrix(viewMatrix)
+            , fShape(shape)
+            , fAA(aa)
+            , fWaited(false) {}
+
+    ~GrMaskUploaderPrepareCallback() override {
+        if (!fWaited) {
+            // This can happen if our owning op list fails to instantiate (so it never prepares)
+            fPixelsReady.wait();
+        }
+    }
+
+    void operator()(GrOpFlushState* flushState) override {
+        TRACE_EVENT0("skia", "Mask Uploader Pre Flush Callback");
+        auto uploadMask = [this](GrDrawOp::WritePixelsFn& writePixelsFn) {
+            TRACE_EVENT0("skia", "Mask Upload");
+            this->fPixelsReady.wait();
+            this->fWaited = true;
+            // If the worker thread was unable to allocate pixels, this check will fail, and we'll
+            // end up drawing with an uninitialized mask texture, but at least we won't crash.
+            if (this->fPixels.addr()) {
+                writePixelsFn(this->fProxy.get(), 0, 0,
+                              this->fPixels.width(), this->fPixels.height(),
+                              kAlpha_8_GrPixelConfig,
+                              this->fPixels.addr(), this->fPixels.rowBytes());
+            }
+        };
+        flushState->addASAPUpload(std::move(uploadMask));
+    }
+
+    SkAutoPixmapStorage* getPixels() { return &fPixels; }
+    SkSemaphore* getSemaphore() { return &fPixelsReady; }
+    const SkIRect& getMaskBounds() const { return fMaskBounds; }
+    const SkMatrix* getViewMatrix() const { return &fViewMatrix; }
+    const GrShape& getShape() const { return fShape; }
+    GrAA getAA() const { return fAA; }
+
+private:
+    // NOTE: This ref cnt isn't thread safe!
+    sk_sp<GrTextureProxy> fProxy;
+    SkAutoPixmapStorage fPixels;
+    SkSemaphore fPixelsReady;
+
+    SkIRect fMaskBounds;
+    SkMatrix fViewMatrix;
+    GrShape fShape;
+    GrAA fAA;
+    bool fWaited;
+};
+
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -205,11 +286,6 @@ bool GrSoftwarePathRenderer::onDrawPath(const DrawPathArgs& args) {
     }
 
     GrUniqueKey maskKey;
-    struct KeyData {
-        SkScalar fFractionalTranslateX;
-        SkScalar fFractionalTranslateY;
-    };
-
     if (useCache) {
         // We require the upper left 2x2 of the matrix to match exactly for a cache hit.
         SkScalar sx = args.fViewMatrix->get(SkMatrix::kMScaleX);
@@ -240,8 +316,6 @@ bool GrSoftwarePathRenderer::onDrawPath(const DrawPathArgs& args) {
         builder[4] = fracX | (fracY >> 8);
         args.fShape->writeUnstyledKey(&builder[5]);
 #endif
-        // FIXME: Doesn't the key need to consider whether we're using AA or not? In practice that
-        // should always be true, though.
     }
 
     sk_sp<GrTextureProxy> proxy;
@@ -251,9 +325,42 @@ bool GrSoftwarePathRenderer::onDrawPath(const DrawPathArgs& args) {
     if (!proxy) {
         SkBackingFit fit = useCache ? SkBackingFit::kExact : SkBackingFit::kApprox;
         GrAA aa = GrAAType::kCoverage == args.fAAType ? GrAA::kYes : GrAA::kNo;
-        proxy = GrSWMaskHelper::DrawShapeMaskToTexture(args.fContext, *args.fShape,
-                                                       *boundsForMask, aa,
-                                                       fit, args.fViewMatrix);
+
+        SkTaskGroup* taskGroup = args.fContext->contextPriv().getTaskGroup();
+        if (taskGroup) {
+            proxy = make_deferred_mask_texture_proxy(args.fContext, fit,
+                                                     boundsForMask->width(),
+                                                     boundsForMask->height());
+            if (!proxy) {
+                return false;
+            }
+
+            auto uploader = skstd::make_unique<GrMaskUploaderPrepareCallback>(
+                    proxy, *boundsForMask, *args.fViewMatrix, *args.fShape, aa);
+            GrMaskUploaderPrepareCallback* uploaderRaw = uploader.get();
+
+            auto drawAndUploadMask = [uploaderRaw] {
+                TRACE_EVENT0("skia", "Threaded SW Mask Render");
+                GrSWMaskHelper helper(uploaderRaw->getPixels());
+                if (helper.init(uploaderRaw->getMaskBounds(), uploaderRaw->getViewMatrix())) {
+                    helper.drawShape(uploaderRaw->getShape(), SkRegion::kReplace_Op,
+                                     uploaderRaw->getAA(), 0xFF);
+                } else {
+                    SkDEBUGFAIL("Unable to allocate SW mask.");
+                }
+                uploaderRaw->getSemaphore()->signal();
+            };
+            taskGroup->add(std::move(drawAndUploadMask));
+            args.fRenderTargetContext->getOpList()->addPrepareCallback(std::move(uploader));
+        } else {
+            GrSWMaskHelper helper;
+            if (!helper.init(*boundsForMask, args.fViewMatrix)) {
+                return false;
+            }
+            helper.drawShape(*args.fShape, SkRegion::kReplace_Op, aa, 0xFF);
+            proxy = helper.toTextureProxy(args.fContext, fit);
+        }
+
         if (!proxy) {
             return false;
         }
diff --git a/tests/TestConfigParsing.cpp b/tests/TestConfigParsing.cpp
index e24c019dc8..8e2276c51a 100644
--- a/tests/TestConfigParsing.cpp
+++ b/tests/TestConfigParsing.cpp
@@ -121,7 +121,8 @@ DEF_TEST(ParseConfigs_DefaultConfigs, reporter) {
         "mock",
         "mtl",
         "gl4444",
-        "gl565"
+        "gl565",
+        "gltestthreading"
     });
 
     SkCommandLineConfigArray configs;
@@ -254,6 +255,8 @@ DEF_TEST(ParseConfigs_DefaultConfigs, reporter) {
     REPORTER_ASSERT(reporter, configs[29]->asConfigGpu()->getColorSpace() == srgbColorSpace.get());
     REPORTER_ASSERT(reporter, configs[30]->asConfigGpu());
     REPORTER_ASSERT(reporter, configs[30]->asConfigGpu()->getSamples() == 4);
+    REPORTER_ASSERT(reporter, configs[47]->asConfigGpu());
+    REPORTER_ASSERT(reporter, configs[47]->asConfigGpu()->getTestThreading());
 #ifdef SK_VULKAN
     REPORTER_ASSERT(reporter, configs[31]->asConfigGpu());
 #endif
diff --git a/tools/flags/SkCommonFlags.cpp b/tools/flags/SkCommonFlags.cpp
index facbd3139b..0986d2b57c 100644
--- a/tools/flags/SkCommonFlags.cpp
+++ b/tools/flags/SkCommonFlags.cpp
@@ -6,6 +6,8 @@
  */
 
 #include "SkCommonFlags.h"
+#include "SkExecutor.h"
+#include "SkOnce.h"
 #include "SkOSFile.h"
 #include "SkOSPath.h"
 
@@ -54,6 +56,9 @@ DEFINE_string(svgs, "", "Directory to read SVGs from, or a single SVG file.");
 DEFINE_int32_2(threads, j, -1, "Run threadsafe tests on a threadpool with this many extra threads, "
                                "defaulting to one extra thread per core.");
 
+DEFINE_int32(gpuThreads, 0, "Create this many extra threads to assist with GPU work, "
+                            "including software path rendering.");
+
 DEFINE_bool2(verbose, v, false, "enable verbose output from the test driver.");
 
 DEFINE_bool2(veryVerbose, V, false, "tell individual tests to be verbose.");
@@ -123,3 +128,9 @@ bool CollectImages(SkCommandLineFlags::StringArray images, SkTArray<SkString>* o
     }
     return true;
 }
+
+SkExecutor* GpuExecutorForTools() {
+    static std::unique_ptr<SkExecutor> gGpuExecutor = (0 != FLAGS_gpuThreads)
+        ? SkExecutor::MakeThreadPool(FLAGS_gpuThreads) : nullptr;
+    return gGpuExecutor.get();
+}
diff --git a/tools/flags/SkCommonFlagsConfig.cpp b/tools/flags/SkCommonFlagsConfig.cpp
index 03b3b96822..16fe914f77 100644
--- a/tools/flags/SkCommonFlagsConfig.cpp
+++ b/tools/flags/SkCommonFlagsConfig.cpp
@@ -72,6 +72,7 @@ static const struct {
     { "glesnarrow",            "gpu", "api=gles,color=f16_narrow" },
     { "gldft",                 "gpu", "api=gl,dit=true" },
     { "glesdft",               "gpu", "api=gles,dit=true" },
+    { "gltestthreading",       "gpu", "api=gl,testThreading=true" },
     { "debuggl",               "gpu", "api=debuggl" },
     { "nullgl",                "gpu", "api=nullgl" },
     { "angle_d3d11_es2",       "gpu", "api=angle_d3d11_es2" },
@@ -170,6 +171,8 @@ static const char configExtendedHelp[] =
     "\t    Use multisampling with N samples.\n"
     "\tstencils\ttype: bool\tdefault: true.\n"
     "\t    Allow the use of stencil buffers.\n"
+    "\ttestThreading\ttype: bool\tdefault: false.\n"
+    "\t    Run config with and without worker threads, check that results match.\n"
     "\n"
     "Predefined configs:\n\n"
     // Help text for pre-defined configs is auto-generated from gPredefinedConfigs
@@ -200,7 +203,7 @@ SkCommandLineConfig::~SkCommandLineConfig() {
 SkCommandLineConfigGpu::SkCommandLineConfigGpu(
     const SkString& tag, const SkTArray<SkString>& viaParts, ContextType contextType, bool useNVPR,
     bool useInstanced, bool useDIText, int samples, SkColorType colorType, SkAlphaType alphaType,
-    sk_sp<SkColorSpace> colorSpace, bool useStencilBuffers)
+    sk_sp<SkColorSpace> colorSpace, bool useStencilBuffers, bool testThreading)
         : SkCommandLineConfig(tag, SkString("gpu"), viaParts)
         , fContextType(contextType)
         , fContextOverrides(ContextOverrides::kNone)
@@ -208,7 +211,8 @@ SkCommandLineConfigGpu::SkCommandLineConfigGpu(
         , fSamples(samples)
         , fColorType(colorType)
         , fAlphaType(alphaType)
-        , fColorSpace(std::move(colorSpace)) {
+        , fColorSpace(std::move(colorSpace))
+        , fTestThreading(testThreading) {
     if (useNVPR) {
         fContextOverrides |= ContextOverrides::kRequireNVPRSupport;
     } else if (!useInstanced) {
@@ -419,6 +423,8 @@ SkCommandLineConfigGpu* parse_command_line_config_gpu(const SkString& tag,
     sk_sp<SkColorSpace> colorSpace = nullptr;
     bool seenUseStencils = false;
     bool useStencils = true;
+    bool seenTestThreading = false;
+    bool testThreading = false;
 
     SkTArray<SkString> optionParts;
     SkStrSplit(options.c_str(), ",", kStrict_SkStrSplitMode, &optionParts);
@@ -452,6 +458,9 @@ SkCommandLineConfigGpu* parse_command_line_config_gpu(const SkString& tag,
         } else if (key.equals("stencils") && !seenUseStencils) {
             valueOk = parse_option_bool(value, &useStencils);
             seenUseStencils = true;
+        } else if (key.equals("testThreading") && !seenTestThreading) {
+            valueOk = parse_option_bool(value, &testThreading);
+            seenTestThreading = true;
         }
         if (!valueOk) {
             return nullptr;
@@ -461,7 +470,8 @@ SkCommandLineConfigGpu* parse_command_line_config_gpu(const SkString& tag,
         return nullptr;
     }
     return new SkCommandLineConfigGpu(tag, vias, contextType, useNVPR, useInstanced, useDIText,
-                                      samples, colorType, alphaType, colorSpace, useStencils);
+                                      samples, colorType, alphaType, colorSpace, useStencils,
+                                      testThreading);
 }
 #endif
 
diff --git a/tools/flags/SkCommonFlagsConfig.h b/tools/flags/SkCommonFlagsConfig.h
index 77f31c32c2..18b138c468 100644
--- a/tools/flags/SkCommonFlagsConfig.h
+++ b/tools/flags/SkCommonFlagsConfig.h
@@ -56,7 +56,8 @@ class SkCommandLineConfigGpu : public SkCommandLineConfig {
     SkCommandLineConfigGpu(const SkString& tag, const SkTArray<SkString>& viaParts,
                            ContextType contextType, bool useNVPR, bool useInstanced, bool useDIText,
                            int samples, SkColorType colorType, SkAlphaType alphaType,
-                           sk_sp<SkColorSpace> colorSpace, bool useStencilBuffers);
+                           sk_sp<SkColorSpace> colorSpace, bool useStencilBuffers,
+                           bool testThreading);
     const SkCommandLineConfigGpu* asConfigGpu() const override { return this; }
     ContextType getContextType() const { return fContextType; }
     ContextOverrides getContextOverrides() const { return fContextOverrides; }
@@ -71,6 +72,7 @@ class SkCommandLineConfigGpu : public SkCommandLineConfig {
     SkColorType getColorType() const { return fColorType; }
     SkAlphaType getAlphaType() const { return fAlphaType; }
     SkColorSpace* getColorSpace() const { return fColorSpace.get(); }
+    bool getTestThreading() const { return fTestThreading; }
 
   private:
     ContextType fContextType;
@@ -80,6 +82,7 @@ class SkCommandLineConfigGpu : public SkCommandLineConfig {
     SkColorType fColorType;
     SkAlphaType fAlphaType;
     sk_sp<SkColorSpace> fColorSpace;
+    bool fTestThreading;
 };
 #endif
 
diff --git a/tools/flags/SkCommonFlagsGpuThreads.h b/tools/flags/SkCommonFlagsGpuThreads.h
new file mode 100644
index 0000000000..a6042fa821
--- /dev/null
+++ b/tools/flags/SkCommonFlagsGpuThreads.h
@@ -0,0 +1,15 @@
+/*
+ * Copyright 2017 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#ifndef SK_COMMON_FLAGS_GPU_THREADS
+#define SK_COMMON_FLAGS_GPU_THREADS
+
+class SkExecutor;
+
+SkExecutor* GpuExecutorForTools();
+
+#endif
diff --git a/tools/viewer/Viewer.cpp b/tools/viewer/Viewer.cpp
index dd8d3303f8..544eaf4c91 100644
--- a/tools/viewer/Viewer.cpp
+++ b/tools/viewer/Viewer.cpp
@@ -19,6 +19,7 @@
 #include "SkColorSpace_Base.h"
 #include "SkColorSpaceXformCanvas.h"
 #include "SkCommandLineFlags.h"
+#include "SkCommonFlagsGpuThreads.h"
 #include "SkCommonFlagsPathRenderer.h"
 #include "SkDashPathEffect.h"
 #include "SkEventTracingPriv.h"
@@ -303,6 +304,7 @@ Viewer::Viewer(int argc, char** argv, void* platformData)
     displayParams.fMSAASampleCount = FLAGS_msaa;
     displayParams.fGrContextOptions.fEnableInstancedRendering = FLAGS_instancedRendering;
     displayParams.fGrContextOptions.fGpuPathRenderers = CollectGpuPathRenderersFromFlags();
+    displayParams.fGrContextOptions.fExecutor = GpuExecutorForTools();
     fWindow->setRequestedDisplayParams(displayParams);
 
     // register callbacks