Make threaded proxy generation MDB-friendly, and defer instantiation

Replaces GrPrepareCallback with GrDeferredProxyUploader, stored directly
on GrTextureProxy. Op lists now store a list of referenced proxies that
are being generated by worker threads. At flush time, iterate over those
proxies, and invoke their uploader.

Lifetime of the uploader object is now tied to the proxy, but the ASAP
upload function will free the proxy's uploader, if it's called.

Bug: skia:
Change-Id: Ieb2c6a805d19990012839a8e103c3ca5b8d3dfc6
Reviewed-on: https://skia-review.googlesource.com/49904
Commit-Queue: Brian Osman <brianosman@google.com>
Reviewed-by: Robert Phillips <robertphillips@google.com>

9 files changed, 180 insertions, 129 deletions

diff --git a/src/gpu/GrClipStackClip.cpp b/src/gpu/GrClipStackClip.cpp
index 10880b4891..309c9cad10 100644
--- a/src/gpu/GrClipStackClip.cpp
+++ b/src/gpu/GrClipStackClip.cpp
@@ -9,11 +9,11 @@
 
 #include "GrAppliedClip.h"
 #include "GrContextPriv.h"
+#include "GrDeferredProxyUploader.h"
 #include "GrDrawingManager.h"
 #include "GrRenderTargetContextPriv.h"
 #include "GrFixedClip.h"
 #include "GrGpuResourcePriv.h"
-#include "GrPrepareCallback.h"
 #include "GrResourceProvider.h"
 #include "GrStencilAttachment.h"
 #include "GrSWMaskHelper.h"
@@ -428,7 +428,7 @@ sk_sp<GrTextureProxy> GrClipStackClip::createAlphaClipMask(GrContext* context,
 namespace {
 
 /**
- * Payload class for use with GrMaskUploaderPrepareCallback. The clip mask code renders multiple
+ * Payload class for use with GrTDeferredProxyUploader. The clip mask code renders multiple
  * elements, each storing their own AA setting (and already transformed into device space). This
  * stores all of the information needed by the worker thread to draw all clip elements (see below,
  * in createSoftwareClipMask).
@@ -526,15 +526,8 @@ sk_sp<GrTextureProxy> GrClipStackClip::createSoftwareClipMask(
         proxy = GrSurfaceProxy::MakeDeferred(context->resourceProvider(), desc,
                                              SkBackingFit::kApprox, SkBudgeted::kYes);
 
-        // TODO: I believe the assignUniqueKeyToProxy below used to instantiate the proxy before
-        // the draw that used the result was being flushed, so the upload was succeeding. With 
-        // assignUniqueKeyToProxy no longer forcing an instantiation it will have to happen
-        // explicitly elsewhere.
-        proxy->instantiate(context->resourceProvider());
-
-        auto uploader = skstd::make_unique<GrMaskUploaderPrepareCallback<ClipMaskData>>(
-                proxy, reducedClip);
-        GrMaskUploaderPrepareCallback<ClipMaskData>* uploaderRaw = uploader.get();
+        auto uploader = skstd::make_unique<GrTDeferredProxyUploader<ClipMaskData>>(reducedClip);
+        GrTDeferredProxyUploader<ClipMaskData>* uploaderRaw = uploader.get();
         auto drawAndUploadMask = [uploaderRaw, maskSpaceIBounds] {
             TRACE_EVENT0("skia", "Threaded SW Clip Mask Render");
             GrSWMaskHelper helper(uploaderRaw->getPixels());
@@ -545,11 +538,11 @@ sk_sp<GrTextureProxy> GrClipStackClip::createSoftwareClipMask(
             } else {
                 SkDEBUGFAIL("Unable to allocate SW clip mask.");
             }
-            uploaderRaw->getSemaphore()->signal();
+            uploaderRaw->signalAndFreeData();
         };
 
         taskGroup->add(std::move(drawAndUploadMask));
-        renderTargetContext->getOpList()->addPrepareCallback(std::move(uploader));
+        proxy->texPriv().setDeferredUploader(std::move(uploader));
     } else {
         GrSWMaskHelper helper;
         if (!helper.init(maskSpaceIBounds)) {
diff --git a/src/gpu/GrDeferredProxyUploader.h b/src/gpu/GrDeferredProxyUploader.h
new file mode 100644
index 0000000000..35581166de
--- /dev/null
+++ b/src/gpu/GrDeferredProxyUploader.h
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2017 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#ifndef GrDeferredProxyUploader_DEFINED
+#define GrDeferredProxyUploader_DEFINED
+
+#include "SkAutoPixmapStorage.h"
+#include "SkMakeUnique.h"
+#include "SkRefCnt.h"
+#include "SkSemaphore.h"
+
+#include "GrOpFlushState.h"
+#include "GrTextureProxyPriv.h"
+
+/**
+ * GrDeferredProxyUploader assists with threaded generation of textures. Currently used by both
+ * software clip masks, and the software path renderer. The calling code typically needs to store
+ * some additional data (T) for use on the worker thread. GrTDeferredProxyUploader allows storing
+ * such data. The common flow is:
+ *
+ * 1) A GrTDeferredProxyUploader is created, with some payload (eg an SkPath to draw).
+ *    The uploader is owned by the proxy that it's going to populate.
+ * 2) A task is created with a pointer to the uploader. A worker thread executes that task, using
+ *    the payload data to allocate and fill in the fPixels pixmap.
+ * 3) The worker thread calls signalAndFreeData(), which notifies the main thread that the pixmap
+ *    is ready, and then deletes the payload data (which is no longer needed).
+ * 4) In parallel to 2-3, on the main thread... Some op is created that refers to the proxy. When
+ *    that op is added to an op list, the op list retains a pointer to the "deferred" proxies.
+ * 5) At flush time, the op list ensures that the deferred proxies are instantiated, then calls
+ *    scheduleUpload on those proxies, which calls scheduleUpload on the uploader (below).
+ * 6) scheduleUpload defers the upload even further, by adding an ASAPUpload to the flush.
+ * 7) When the ASAP upload happens, we wait to make sure that the pixels are marked ready
+ *    (from step #3 on the worker thread). Then we perform the actual upload to the texture.
+ *    Finally, we call resetDeferredUploader, which deletes the uploader object, causing fPixels
+ *    to be freed.
+ */
+class GrDeferredProxyUploader : public SkNoncopyable {
+public:
+    GrDeferredProxyUploader() : fScheduledUpload(false), fWaited(false) {}
+
+    virtual ~GrDeferredProxyUploader() {
+        if (!fWaited) {
+            // This can happen if our owning proxy fails to instantiate
+            fPixelsReady.wait();
+        }
+    }
+
+    void scheduleUpload(GrOpFlushState* flushState, GrTextureProxy* proxy) {
+        if (fScheduledUpload) {
+            // Multiple references to the owning proxy may have caused us to already execute
+            return;
+        }
+
+        auto uploadMask = [this, proxy](GrDrawOp::WritePixelsFn& writePixelsFn) {
+            this->fPixelsReady.wait();
+            this->fWaited = true;
+            // If the worker thread was unable to allocate pixels, this check will fail, and we'll
+            // end up drawing with an uninitialized mask texture, but at least we won't crash.
+            if (this->fPixels.addr()) {
+                writePixelsFn(proxy, 0, 0, this->fPixels.width(), this->fPixels.height(),
+                              proxy->config(), this->fPixels.addr(), this->fPixels.rowBytes());
+            }
+            // Upload has finished, so tell the proxy to release this GrDeferredProxyUploader
+            proxy->texPriv().resetDeferredUploader();
+        };
+        flushState->addASAPUpload(std::move(uploadMask));
+        fScheduledUpload = true;
+    }
+
+    void signalAndFreeData() {
+        fPixelsReady.signal();
+        this->freeData();
+    }
+
+    SkAutoPixmapStorage* getPixels() { return &fPixels; }
+
+private:
+    virtual void freeData() {}
+
+    SkAutoPixmapStorage fPixels;
+    SkSemaphore fPixelsReady;
+    bool fScheduledUpload;
+    bool fWaited;
+};
+
+template <typename T>
+class GrTDeferredProxyUploader : public GrDeferredProxyUploader {
+public:
+    template <typename... Args>
+    GrTDeferredProxyUploader(Args&&... args)
+        : fData(skstd::make_unique<T>(std::forward<Args>(args)...)) {
+    }
+
+    T& data() { return *fData; }
+
+private:
+    void freeData() override {
+        fData.reset();
+    }
+
+    std::unique_ptr<T> fData;
+};
+
+#endif
diff --git a/src/gpu/GrDrawingManager.cpp b/src/gpu/GrDrawingManager.cpp
index 013c624efa..5deacbb5ca 100644
--- a/src/gpu/GrDrawingManager.cpp
+++ b/src/gpu/GrDrawingManager.cpp
@@ -195,6 +195,8 @@ GrSemaphoresSubmitted GrDrawingManager::internalFlush(GrSurfaceProxy*,
             continue;
         }
 
+        // Instantiate all deferred proxies (being built on worker threads) so we can upload them
+        fOpLists[i]->instantiateDeferredProxies(fContext->resourceProvider());
         fOpLists[i]->prepare(&fFlushState);
     }
 
diff --git a/src/gpu/GrOpList.cpp b/src/gpu/GrOpList.cpp
index 65d4e73fd0..59e1cd0f23 100644
--- a/src/gpu/GrOpList.cpp
+++ b/src/gpu/GrOpList.cpp
@@ -8,8 +8,9 @@
 #include "GrOpList.h"
 
 #include "GrContext.h"
-#include "GrPrepareCallback.h"
+#include "GrDeferredProxyUploader.h"
 #include "GrSurfaceProxy.h"
+#include "GrTextureProxyPriv.h"
 
 #include "SkAtomics.h"
 
@@ -57,17 +58,19 @@ void GrOpList::reset() {
     }
 
     fTarget.reset();
-    fPrepareCallbacks.reset();
+    fDeferredProxies.reset();
     fAuditTrail = nullptr;
 }
 
-void GrOpList::addPrepareCallback(std::unique_ptr<GrPrepareCallback> callback) {
-    fPrepareCallbacks.push_back(std::move(callback));
+void GrOpList::instantiateDeferredProxies(GrResourceProvider* resourceProvider) {
+    for (int i = 0; i < fDeferredProxies.count(); ++i) {
+        fDeferredProxies[i]->instantiate(resourceProvider);
+    }
 }
 
 void GrOpList::prepare(GrOpFlushState* flushState) {
-    for (int i = 0; i < fPrepareCallbacks.count(); ++i) {
-        (*fPrepareCallbacks[i])(flushState);
+    for (int i = 0; i < fDeferredProxies.count(); ++i) {
+        fDeferredProxies[i]->texPriv().scheduleUpload(flushState);
     }
 
     this->onPrepare(flushState);
@@ -100,6 +103,12 @@ void GrOpList::addDependency(GrSurfaceProxy* dependedOn, const GrCaps& caps) {
             opList->makeClosed(caps);
         }
     }
+
+    if (GrTextureProxy* textureProxy = dependedOn->asTextureProxy()) {
+        if (textureProxy->texPriv().isDeferred()) {
+            fDeferredProxies.push_back(textureProxy);
+        }
+    }
 }
 
 #ifdef SK_DEBUG
diff --git a/src/gpu/GrOpList.h b/src/gpu/GrOpList.h
index 7a97961b19..6322008f25 100644
--- a/src/gpu/GrOpList.h
+++ b/src/gpu/GrOpList.h
@@ -23,7 +23,6 @@
 class GrAuditTrail;
 class GrCaps;
 class GrOpFlushState;
-class GrPrepareCallback;
 class GrRenderTargetOpList;
 class GrResourceAllocator;
 class GrResourceProvider;
@@ -39,8 +38,10 @@ public:
     GrOpList(GrResourceProvider*, GrSurfaceProxy*, GrAuditTrail*);
     ~GrOpList() override;
 
-    // These three methods are invoked at flush time
+    // These four methods are invoked at flush time
     bool instantiate(GrResourceProvider* resourceProvider);
+    // Instantiates any "threaded" texture proxies that are being prepared elsewhere
+    void instantiateDeferredProxies(GrResourceProvider* resourceProvider);
     void prepare(GrOpFlushState* flushState);
     bool execute(GrOpFlushState* flushState) { return this->onExecute(flushState); }
 
@@ -59,8 +60,6 @@ public:
 
     virtual void reset();
 
-    void addPrepareCallback(std::unique_ptr<GrPrepareCallback> callback);
-
     // TODO: in an MDB world, where the OpLists don't allocate GPU resources, it seems like
     // these could go away
     virtual void abandonGpuResources() = 0;
@@ -181,8 +180,8 @@ private:
     // 'this' GrOpList relies on the output of the GrOpLists in 'fDependencies'
     SkSTArray<1, GrOpList*, true> fDependencies;
 
-    // These are used rarely, most clients never produce any
-    SkTArray<std::unique_ptr<GrPrepareCallback>> fPrepareCallbacks;
+    // List of texture proxies whose contents are being prepared on a worker thread
+    SkTArray<GrTextureProxy*, true> fDeferredProxies;
 
     typedef SkRefCnt INHERITED;
 };
diff --git a/src/gpu/GrPrepareCallback.h b/src/gpu/GrPrepareCallback.h
deleted file mode 100644
index af8161d163..0000000000
--- a/src/gpu/GrPrepareCallback.h
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright 2017 Google Inc.
- *
- * Use of this source code is governed by a BSD-style license that can be
- * found in the LICENSE file.
- */
-
-#ifndef GrPrepareCallback_DEFINED
-#define GrPrepareCallback_DEFINED
-
-#include "SkAutoPixmapStorage.h"
-#include "SkRefCnt.h"
-#include "SkSemaphore.h"
-
-#include "GrOpFlushState.h"
-
-class GrTextureProxy;
-
-/**
- * An instance of any class derived from GrPrepareCallback can be passed to
- * GrOpList::addPrepareCallback. At flush time, all callbacks (on op lists being flushed) will be
- * invoked (via operator()). Note that the callback receives the GrOpFlushState, so it can trigger
- * ASAP uploads (similar to an Op's onPrepare).
- *
- * All callbacks are invoked at the beginning of flush, before prepare is called.
- */
-class GrPrepareCallback : SkNoncopyable {
-public:
-    virtual ~GrPrepareCallback() {}
-    virtual void operator()(GrOpFlushState*) = 0;
-};
-
-/**
- * GrMaskUploaderPrepareCallback assists with threaded generation of mask textures. Currently used
- * by both software clip masks, and the software path renderer. The calling code typically needs
- * to store some additional data (T) for use on the worker thread. That payload is accessed by the
- * worker thread to populate the mask in fPixels (using GrSWMaskHelper). This callback's operator()
- * handles scheduling the texture upload at flush time.
- */
-template <typename T>
-class GrMaskUploaderPrepareCallback : public GrPrepareCallback {
-public:
-    template <typename... Args>
-    GrMaskUploaderPrepareCallback(sk_sp<GrTextureProxy> proxy, Args&&... args)
-            : fProxy(std::move(proxy))
-            , fWaited(false)
-            , fData(std::forward<Args>(args)...) {}
-
-    ~GrMaskUploaderPrepareCallback() override {
-        if (!fWaited) {
-            // This can happen if our owning op list fails to instantiate (so it never prepares)
-            fPixelsReady.wait();
-        }
-    }
-
-    void operator()(GrOpFlushState* flushState) override {
-        auto uploadMask = [this](GrDrawOp::WritePixelsFn& writePixelsFn) {
-            this->fPixelsReady.wait();
-            this->fWaited = true;
-            // If the worker thread was unable to allocate pixels, this check will fail, and we'll
-            // end up drawing with an uninitialized mask texture, but at least we won't crash.
-            if (this->fPixels.addr()) {
-                writePixelsFn(this->fProxy.get(), 0, 0,
-                              this->fPixels.width(), this->fPixels.height(),
-                              kAlpha_8_GrPixelConfig,
-                              this->fPixels.addr(), this->fPixels.rowBytes());
-                // Free this memory immediately, so it can be recycled. This avoids memory pressure
-                // when there is a large amount of threaded work still running during flush.
-                this->fPixels.reset();
-            }
-        };
-        flushState->addASAPUpload(std::move(uploadMask));
-    }
-
-    SkAutoPixmapStorage* getPixels() { return &fPixels; }
-    SkSemaphore* getSemaphore() { return &fPixelsReady; }
-    T& data() { return fData; }
-
-private:
-    sk_sp<GrTextureProxy> fProxy;
-    SkAutoPixmapStorage fPixels;
-    SkSemaphore fPixelsReady;
-    bool fWaited;
-
-    T fData;
-};
-
-#endif
diff --git a/src/gpu/GrSoftwarePathRenderer.cpp b/src/gpu/GrSoftwarePathRenderer.cpp
index 9f876f1213..cd37dffc39 100644
--- a/src/gpu/GrSoftwarePathRenderer.cpp
+++ b/src/gpu/GrSoftwarePathRenderer.cpp
@@ -9,10 +9,10 @@
 #include "GrAuditTrail.h"
 #include "GrClip.h"
 #include "GrContextPriv.h"
+#include "GrDeferredProxyUploader.h"
 #include "GrGpuResourcePriv.h"
 #include "GrOpFlushState.h"
 #include "GrOpList.h"
-#include "GrPrepareCallback.h"
 #include "GrResourceProvider.h"
 #include "GrSWMaskHelper.h"
 #include "SkMakeUnique.h"
@@ -175,7 +175,7 @@ static sk_sp<GrTextureProxy> make_deferred_mask_texture_proxy(GrContext* context
 namespace {
 
 /**
- * Payload class for use with GrMaskUploaderPrepareCallback. The software path renderer only draws
+ * Payload class for use with GrTDeferredProxyUploader. The software path renderer only draws
  * a single path into the mask texture. This stores all of the information needed by the worker
  * thread's call to drawShape (see below, in onDrawPath).
  */
@@ -317,15 +317,9 @@ bool GrSoftwarePathRenderer::onDrawPath(const DrawPathArgs& args) {
                 return false;
             }
 
-            // TODO: I believe the assignUniqueKeyToProxy below used to instantiate the proxy before
-            // before the draw that used the result was being flushed, so the upload was succeeding.
-            // With assignUniqueKeyToProxy no longer forcing an instantiation it will have to happen
-            // explicitly elsewhere.
-            proxy->instantiate(fResourceProvider);
-
-            auto uploader = skstd::make_unique<GrMaskUploaderPrepareCallback<SoftwarePathData>>(
-                    proxy, *boundsForMask, *args.fViewMatrix, *args.fShape, aa);
-            GrMaskUploaderPrepareCallback<SoftwarePathData>* uploaderRaw = uploader.get();
+            auto uploader = skstd::make_unique<GrTDeferredProxyUploader<SoftwarePathData>>(
+                    *boundsForMask, *args.fViewMatrix, *args.fShape, aa);
+            GrTDeferredProxyUploader<SoftwarePathData>* uploaderRaw = uploader.get();
 
             auto drawAndUploadMask = [uploaderRaw] {
                 TRACE_EVENT0("skia", "Threaded SW Mask Render");
@@ -337,10 +331,10 @@ bool GrSoftwarePathRenderer::onDrawPath(const DrawPathArgs& args) {
                 } else {
                     SkDEBUGFAIL("Unable to allocate SW mask.");
                 }
-                uploaderRaw->getSemaphore()->signal();
+                uploaderRaw->signalAndFreeData();
             };
             taskGroup->add(std::move(drawAndUploadMask));
-            args.fRenderTargetContext->getOpList()->addPrepareCallback(std::move(uploader));
+            proxy->texPriv().setDeferredUploader(std::move(uploader));
         } else {
             GrSWMaskHelper helper;
             if (!helper.init(*boundsForMask)) {
diff --git a/src/gpu/GrTextureProxy.cpp b/src/gpu/GrTextureProxy.cpp
index 8b9e58b594..534adaa709 100644
--- a/src/gpu/GrTextureProxy.cpp
+++ b/src/gpu/GrTextureProxy.cpp
@@ -6,10 +6,11 @@
  */
 
 #include "GrTextureProxy.h"
+#include "GrTextureProxyPriv.h"
 
 #include "GrContext.h"
+#include "GrDeferredProxyUploader.h"
 #include "GrResourceCache.h"
-
 #include "GrTexturePriv.h"
 
 GrTextureProxy::GrTextureProxy(const GrSurfaceDesc& srcDesc, SkBackingFit fit, SkBudgeted budgeted,
@@ -17,7 +18,8 @@ GrTextureProxy::GrTextureProxy(const GrSurfaceDesc& srcDesc, SkBackingFit fit, S
         : INHERITED(srcDesc, fit, budgeted, flags)
         , fIsMipMapped(false)
         , fMipColorMode(SkDestinationSurfaceColorMode::kLegacy)
-        , fCache(nullptr) {
+        , fCache(nullptr)
+        , fDeferredUploader(nullptr) {
     SkASSERT(!srcData);  // currently handled in Make()
 }
 
@@ -25,7 +27,8 @@ GrTextureProxy::GrTextureProxy(sk_sp<GrSurface> surf, GrSurfaceOrigin origin)
         : INHERITED(std::move(surf), origin, SkBackingFit::kExact)
         , fIsMipMapped(fTarget->asTexture()->texturePriv().hasMipMaps())
         , fMipColorMode(fTarget->asTexture()->texturePriv().mipColorMode())
-        , fCache(nullptr) {
+        , fCache(nullptr)
+        , fDeferredUploader(nullptr) {
     if (fTarget->getUniqueKey().isValid()) {
         fCache = fTarget->asTexture()->getContext()->getResourceCache();
         fCache->adoptUniqueKeyFromSurface(this, fTarget);
@@ -67,6 +70,25 @@ sk_sp<GrSurface> GrTextureProxy::createSurface(GrResourceProvider* resourceProvi
     return surface;
 }
 
+void GrTextureProxyPriv::setDeferredUploader(std::unique_ptr<GrDeferredProxyUploader> uploader) {
+    SkASSERT(!fTextureProxy->fDeferredUploader);
+    fTextureProxy->fDeferredUploader = std::move(uploader);
+}
+
+void GrTextureProxyPriv::scheduleUpload(GrOpFlushState* flushState) {
+    SkASSERT(fTextureProxy->fDeferredUploader);
+
+    // Instantiate might have failed
+    if (fTextureProxy->fTarget) {
+        fTextureProxy->fDeferredUploader->scheduleUpload(flushState, fTextureProxy);
+    }
+}
+
+void GrTextureProxyPriv::resetDeferredUploader() {
+    SkASSERT(fTextureProxy->fDeferredUploader);
+    fTextureProxy->fDeferredUploader.reset();
+}
+
 // This method parallels the highest_filter_mode functions in GrGLTexture & GrVkTexture.
 GrSamplerState::Filter GrTextureProxy::highestFilterMode() const {
     if (fTarget) {
diff --git a/src/gpu/GrTextureProxyPriv.h b/src/gpu/GrTextureProxyPriv.h
index c3ddb9460c..e961493179 100644
--- a/src/gpu/GrTextureProxyPriv.h
+++ b/src/gpu/GrTextureProxyPriv.h
@@ -10,11 +10,23 @@
 
 #include "GrTextureProxy.h"
 
+class GrDeferredProxyUploader;
+class GrOpFlushState;
+
 /**
  * This class hides the more specialized capabilities of GrTextureProxy.
  */
 class GrTextureProxyPriv {
 public:
+    // Attach a deferred uploader to the proxy. Holds data being prepared by a worker thread.
+    void setDeferredUploader(std::unique_ptr<GrDeferredProxyUploader>);
+    bool isDeferred() const { return SkToBool(fTextureProxy->fDeferredUploader.get()); }
+    // For a deferred proxy (one that has a deferred uploader attached), this schedules an ASAP
+    // upload of that data to the instantiated texture.
+    void scheduleUpload(GrOpFlushState*);
+    // Clears any deferred uploader object on the proxy. Used to free the CPU data after the
+    // contents have been uploaded.
+    void resetDeferredUploader();
 
 private:
     explicit GrTextureProxyPriv(GrTextureProxy* textureProxy) : fTextureProxy(textureProxy) {}