Add resource tracking output and command buffer recycling

BUG=skia:5042
GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2115993002

Review-Url: https://codereview.chromium.org/2115993002

16 files changed, 221 insertions, 45 deletions

diff --git a/src/gpu/vk/GrVkBuffer.h b/src/gpu/vk/GrVkBuffer.h
index 985b586336..bb053ccce0 100644
--- a/src/gpu/vk/GrVkBuffer.h
+++ b/src/gpu/vk/GrVkBuffer.h
@@ -57,6 +57,11 @@ protected:
         Resource(VkBuffer buf, const GrVkAlloc& alloc, Type type)
             : INHERITED(), fBuffer(buf), fAlloc(alloc), fType(type) {}
 
+#ifdef SK_TRACE_VK_RESOURCES
+        void dumpInfo() const override {
+            SkDebugf("GrVkBuffer: %d (%d refs)\n", fBuffer, this->getRefCnt());
+        }
+#endif
         VkBuffer           fBuffer;
         GrVkAlloc          fAlloc;
         Type               fType;
diff --git a/src/gpu/vk/GrVkCommandBuffer.cpp b/src/gpu/vk/GrVkCommandBuffer.cpp
index f076c0b6f1..7fb143ed98 100644
--- a/src/gpu/vk/GrVkCommandBuffer.cpp
+++ b/src/gpu/vk/GrVkCommandBuffer.cpp
@@ -52,6 +52,22 @@ void GrVkCommandBuffer::abandonSubResources() const {
     }
 }
 
+void GrVkCommandBuffer::reset(GrVkGpu* gpu) {
+    SkASSERT(!fIsActive);
+    for (int i = 0; i < fTrackedResources.count(); ++i) {
+        fTrackedResources[i]->unref(gpu);
+    }
+    fTrackedResources.reset();
+
+    this->invalidateState();
+
+    // we will retain resources for later use
+    VkCommandBufferResetFlags flags = 0;
+    GR_VK_CALL(gpu->vkInterface(), ResetCommandBuffer(fCmdBuffer, flags));
+
+    this->onReset(gpu);
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // CommandBuffer commands
 ////////////////////////////////////////////////////////////////////////////////
@@ -320,13 +336,14 @@ void GrVkPrimaryCommandBuffer::endRenderPass(const GrVkGpu* gpu) {
 }
 
 void GrVkPrimaryCommandBuffer::executeCommands(const GrVkGpu* gpu,
-                                               const GrVkSecondaryCommandBuffer* buffer) {
+                                               GrVkSecondaryCommandBuffer* buffer) {
     SkASSERT(fIsActive);
     SkASSERT(fActiveRenderPass);
     SkASSERT(fActiveRenderPass->isCompatible(*buffer->fActiveRenderPass));
 
     GR_VK_CALL(gpu->vkInterface(), CmdExecuteCommands(fCmdBuffer, 1, &buffer->fCmdBuffer));
-    this->addResource(buffer);
+    buffer->ref();
+    fSecondaryCommandBuffers.push_back(buffer);
     // When executing a secondary command buffer all state (besides render pass state) becomes
     // invalidated and must be reset. This includes bound buffers, pipelines, dynamic state, etc.
     this->invalidateState();
@@ -338,12 +355,16 @@ void GrVkPrimaryCommandBuffer::submitToQueue(const GrVkGpu* gpu,
     SkASSERT(!fIsActive);
 
     VkResult err;
-    VkFenceCreateInfo fenceInfo;
-    memset(&fenceInfo, 0, sizeof(VkFenceCreateInfo));
-    fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
-    err = GR_VK_CALL(gpu->vkInterface(), CreateFence(gpu->device(), &fenceInfo, nullptr,
-                                                     &fSubmitFence));
-    SkASSERT(!err);
+    if (VK_NULL_HANDLE == fSubmitFence) {
+        VkFenceCreateInfo fenceInfo;
+        memset(&fenceInfo, 0, sizeof(VkFenceCreateInfo));
+        fenceInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
+        err = GR_VK_CALL(gpu->vkInterface(), CreateFence(gpu->device(), &fenceInfo, nullptr,
+                                                         &fSubmitFence));
+        SkASSERT(!err);
+    } else {
+        GR_VK_CALL(gpu->vkInterface(), ResetFences(gpu->device(), 1, &fSubmitFence));
+    }
 
     VkSubmitInfo submitInfo;
     memset(&submitInfo, 0, sizeof(VkSubmitInfo));
@@ -395,6 +416,13 @@ bool GrVkPrimaryCommandBuffer::finished(const GrVkGpu* gpu) const {
     return false;
 }
 
+void GrVkPrimaryCommandBuffer::onReset(GrVkGpu* gpu) {
+    for (int i = 0; i < fSecondaryCommandBuffers.count(); ++i) {
+        gpu->resourceProvider().recycleSecondaryCommandBuffer(fSecondaryCommandBuffers[i]);
+    }
+    fSecondaryCommandBuffers.reset();
+}
+
 void GrVkPrimaryCommandBuffer::copyImage(const GrVkGpu* gpu,
                                          GrVkImage* srcImage,
                                          VkImageLayout srcLayout,
@@ -538,10 +566,8 @@ void GrVkPrimaryCommandBuffer::onFreeGPUData(const GrVkGpu* gpu) const {
 // SecondaryCommandBuffer
 ////////////////////////////////////////////////////////////////////////////////
 
-GrVkSecondaryCommandBuffer* GrVkSecondaryCommandBuffer::Create(
-                                                       const GrVkGpu* gpu,
-                                                       VkCommandPool cmdPool,
-                                                       const GrVkRenderPass* compatibleRenderPass) {
+GrVkSecondaryCommandBuffer* GrVkSecondaryCommandBuffer::Create(const GrVkGpu* gpu,
+                                                               VkCommandPool cmdPool) {
     const VkCommandBufferAllocateInfo cmdInfo = {
         VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,   // sType
         NULL,                                             // pNext
@@ -557,13 +583,15 @@ GrVkSecondaryCommandBuffer* GrVkSecondaryCommandBuffer::Create(
     if (err) {
         return nullptr;
     }
-    return new GrVkSecondaryCommandBuffer(cmdBuffer, compatibleRenderPass);
+    return new GrVkSecondaryCommandBuffer(cmdBuffer);
 }
 
 
-void GrVkSecondaryCommandBuffer::begin(const GrVkGpu* gpu, const GrVkFramebuffer* framebuffer) {
+void GrVkSecondaryCommandBuffer::begin(const GrVkGpu* gpu, const GrVkFramebuffer* framebuffer,
+                                       const GrVkRenderPass* compatibleRenderPass) {
     SkASSERT(!fIsActive);
-    SkASSERT(fActiveRenderPass);
+    SkASSERT(compatibleRenderPass);
+    fActiveRenderPass = compatibleRenderPass;
 
     VkCommandBufferInheritanceInfo inheritanceInfo;
     memset(&inheritanceInfo, 0, sizeof(VkCommandBufferInheritanceInfo));
diff --git a/src/gpu/vk/GrVkCommandBuffer.h b/src/gpu/vk/GrVkCommandBuffer.h
index 95935f0908..525ea776bb 100644
--- a/src/gpu/vk/GrVkCommandBuffer.h
+++ b/src/gpu/vk/GrVkCommandBuffer.h
@@ -117,6 +117,8 @@ public:
         fTrackedResources.push_back(resource);
     }
 
+    void reset(GrVkGpu* gpu);
+
 protected:
         GrVkCommandBuffer(VkCommandBuffer cmdBuffer, const GrVkRenderPass* rp = VK_NULL_HANDLE)
             : fTrackedResources(kInitialTrackedResourcesCount)
@@ -147,6 +149,8 @@ private:
     virtual void onFreeGPUData(const GrVkGpu* gpu) const = 0;
     void abandonSubResources() const override;
 
+    virtual void onReset(GrVkGpu* gpu) {}
+
     VkBuffer                                fBoundVertexBuffer;
     bool                                    fBoundVertexBufferIsValid;
 
@@ -185,7 +189,7 @@ public:
     // currently inside a render pass that is compatible with the one used to create the
     // SecondaryCommandBuffer.
     void executeCommands(const GrVkGpu* gpu,
-                         const GrVkSecondaryCommandBuffer* secondaryBuffer);
+                         GrVkSecondaryCommandBuffer* secondaryBuffer);
 
     // Commands that only work outside of a render pass
     void clearColorImage(const GrVkGpu* gpu,
@@ -260,6 +264,12 @@ public:
     void submitToQueue(const GrVkGpu* gpu, VkQueue queue, GrVkGpu::SyncQueue sync);
     bool finished(const GrVkGpu* gpu) const;
 
+#ifdef SK_TRACE_VK_RESOURCES
+    void dumpInfo() const override {
+        SkDebugf("GrVkPrimaryCommandBuffer: %d (%d refs)\n", fCmdBuffer, this->getRefCnt());
+    }
+#endif
+
 private:
     explicit GrVkPrimaryCommandBuffer(VkCommandBuffer cmdBuffer)
         : INHERITED(cmdBuffer)
@@ -267,23 +277,31 @@ private:
 
     void onFreeGPUData(const GrVkGpu* gpu) const override;
 
-    VkFence                   fSubmitFence;
+    void onReset(GrVkGpu* gpu) override;
+
+    SkTArray<GrVkSecondaryCommandBuffer*, true> fSecondaryCommandBuffers;
+    VkFence                                     fSubmitFence;
 
     typedef GrVkCommandBuffer INHERITED;
 };
 
 class GrVkSecondaryCommandBuffer : public GrVkCommandBuffer {
 public:
-    static GrVkSecondaryCommandBuffer* Create(const GrVkGpu* gpu, VkCommandPool cmdPool, 
-                                              const GrVkRenderPass* compatibleRenderPass);
+    static GrVkSecondaryCommandBuffer* Create(const GrVkGpu* gpu, VkCommandPool cmdPool);
 
-    void begin(const GrVkGpu* gpu, const GrVkFramebuffer* framebuffer);
+    void begin(const GrVkGpu* gpu, const GrVkFramebuffer* framebuffer,
+               const GrVkRenderPass* compatibleRenderPass);
     void end(const GrVkGpu* gpu);
 
+#ifdef SK_TRACE_VK_RESOURCES
+    void dumpInfo() const override {
+        SkDebugf("GrVkSecondaryCommandBuffer: %d (%d refs)\n", fCmdBuffer, this->getRefCnt());
+    }
+#endif
+
 private:
-    explicit GrVkSecondaryCommandBuffer(VkCommandBuffer cmdBuffer,
-                                        const GrVkRenderPass* compatibleRenderPass)
-        : INHERITED(cmdBuffer, compatibleRenderPass) {
+    explicit GrVkSecondaryCommandBuffer(VkCommandBuffer cmdBuffer)
+        : INHERITED(cmdBuffer) {
     }
 
     void onFreeGPUData(const GrVkGpu* gpu) const override {}
diff --git a/src/gpu/vk/GrVkDescriptorPool.h b/src/gpu/vk/GrVkDescriptorPool.h
index 8fcc67086c..5327a7de8c 100644
--- a/src/gpu/vk/GrVkDescriptorPool.h
+++ b/src/gpu/vk/GrVkDescriptorPool.h
@@ -31,6 +31,13 @@ public:
     // not in use by another draw, to support the requested type and count.
     bool isCompatible(VkDescriptorType type, uint32_t count) const;
 
+#ifdef SK_TRACE_VK_RESOURCES
+    void dumpInfo() const override {
+        SkDebugf("GrVkDescriptorPool: %d, type %d (%d refs)\n", fDescPool, fType, 
+                 this->getRefCnt());
+    }
+#endif
+
 private:
     void freeGPUData(const GrVkGpu* gpu) const override;
 
diff --git a/src/gpu/vk/GrVkFramebuffer.h b/src/gpu/vk/GrVkFramebuffer.h
index 0f8a1c5282..e16ac225c5 100644
--- a/src/gpu/vk/GrVkFramebuffer.h
+++ b/src/gpu/vk/GrVkFramebuffer.h
@@ -29,6 +29,12 @@ public:
 
     VkFramebuffer framebuffer() const { return fFramebuffer; }
 
+#ifdef SK_TRACE_VK_RESOURCES
+    void dumpInfo() const override {
+        SkDebugf("GrVkFramebuffer: %d (%d refs)\n", fFramebuffer, this->getRefCnt());
+    }
+#endif
+
 private:
     GrVkFramebuffer(VkFramebuffer framebuffer) : INHERITED(), fFramebuffer(framebuffer) {}
 
diff --git a/src/gpu/vk/GrVkGpu.cpp b/src/gpu/vk/GrVkGpu.cpp
index e907d8d878..6875824f45 100644
--- a/src/gpu/vk/GrVkGpu.cpp
+++ b/src/gpu/vk/GrVkGpu.cpp
@@ -127,17 +127,18 @@ GrVkGpu::GrVkGpu(GrContext* context, const GrContextOptions& options,
     VK_CALL(GetPhysicalDeviceMemoryProperties(backendCtx->fPhysicalDevice, &fPhysDevMemProps));
 
     const VkCommandPoolCreateInfo cmdPoolInfo = {
-        VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, // sType
-        nullptr,                                    // pNext
-        VK_COMMAND_POOL_CREATE_TRANSIENT_BIT,       // CmdPoolCreateFlags
-        backendCtx->fGraphicsQueueIndex,            // queueFamilyIndex
+        VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,      // sType
+        nullptr,                                         // pNext
+        VK_COMMAND_POOL_CREATE_TRANSIENT_BIT |
+        VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, // CmdPoolCreateFlags
+        backendCtx->fGraphicsQueueIndex,                 // queueFamilyIndex
     };
     GR_VK_CALL_ERRCHECK(this->vkInterface(), CreateCommandPool(fDevice, &cmdPoolInfo, nullptr,
                                                                &fCmdPool));
 
     // must call this after creating the CommandPool
     fResourceProvider.init();
-    fCurrentCmdBuffer = fResourceProvider.createPrimaryCommandBuffer();
+    fCurrentCmdBuffer = fResourceProvider.findOrCreatePrimaryCommandBuffer();
     SkASSERT(fCurrentCmdBuffer);
     fCurrentCmdBuffer->begin(this);
 
@@ -217,7 +218,7 @@ void GrVkGpu::submitCommandBuffer(SyncQueue sync) {
 
     // Release old command buffer and create a new one
     fCurrentCmdBuffer->unref(this);
-    fCurrentCmdBuffer = fResourceProvider.createPrimaryCommandBuffer();
+    fCurrentCmdBuffer = fResourceProvider.findOrCreatePrimaryCommandBuffer();
     SkASSERT(fCurrentCmdBuffer);
 
     fCurrentCmdBuffer->begin(this);
@@ -1495,7 +1496,7 @@ bool GrVkGpu::onReadPixels(GrSurface* surface,
     return true;
 }
 
-void GrVkGpu::submitSecondaryCommandBuffer(const GrVkSecondaryCommandBuffer* buffer,
+void GrVkGpu::submitSecondaryCommandBuffer(GrVkSecondaryCommandBuffer* buffer,
                                            const GrVkRenderPass* renderPass,
                                            const VkClearValue* colorClear,
                                            GrVkRenderTarget* target,
diff --git a/src/gpu/vk/GrVkGpu.h b/src/gpu/vk/GrVkGpu.h
index 542917e9d0..df1aae1824 100644
--- a/src/gpu/vk/GrVkGpu.h
+++ b/src/gpu/vk/GrVkGpu.h
@@ -130,7 +130,7 @@ public:
     }
 #endif
 
-    void submitSecondaryCommandBuffer(const GrVkSecondaryCommandBuffer*,
+    void submitSecondaryCommandBuffer(GrVkSecondaryCommandBuffer*,
                                       const GrVkRenderPass*,
                                       const VkClearValue*,
                                       GrVkRenderTarget*,
diff --git a/src/gpu/vk/GrVkGpuCommandBuffer.cpp b/src/gpu/vk/GrVkGpuCommandBuffer.cpp
index 2eb1671269..8ee83511a6 100644
--- a/src/gpu/vk/GrVkGpuCommandBuffer.cpp
+++ b/src/gpu/vk/GrVkGpuCommandBuffer.cpp
@@ -84,8 +84,8 @@ GrVkGpuCommandBuffer::GrVkGpuCommandBuffer(GrVkGpu* gpu,
 
     GrColorToRGBAFloat(colorInfo.fClearColor, fColorClearValue.color.float32);
 
-    fCommandBuffer = GrVkSecondaryCommandBuffer::Create(gpu, gpu->cmdPool(), fRenderPass);
-    fCommandBuffer->begin(gpu, target->framebuffer());
+    fCommandBuffer = gpu->resourceProvider().findOrCreateSecondaryCommandBuffer();
+    fCommandBuffer->begin(gpu, target->framebuffer(), fRenderPass);
 }
 
 GrVkGpuCommandBuffer::~GrVkGpuCommandBuffer() {
diff --git a/src/gpu/vk/GrVkImage.h b/src/gpu/vk/GrVkImage.h
index fe180698e8..2bfc08d150 100644
--- a/src/gpu/vk/GrVkImage.h
+++ b/src/gpu/vk/GrVkImage.h
@@ -107,6 +107,11 @@ private:
 
         ~Resource() override {}
 
+#ifdef SK_TRACE_VK_RESOURCES
+        void dumpInfo() const override {
+            SkDebugf("GrVkImage: %d (%d refs)\n", fImage, this->getRefCnt()); 
+        }
+#endif
     private:
         void freeGPUData(const GrVkGpu* gpu) const override;
 
diff --git a/src/gpu/vk/GrVkImageView.h b/src/gpu/vk/GrVkImageView.h
index 4be0d199c6..1398987f8b 100644
--- a/src/gpu/vk/GrVkImageView.h
+++ b/src/gpu/vk/GrVkImageView.h
@@ -26,6 +26,12 @@ public:
 
     VkImageView imageView() const { return fImageView; }
 
+#ifdef SK_TRACE_VK_RESOURCES
+    void dumpInfo() const override {
+        SkDebugf("GrVkImageView: %d (%d refs)\n", fImageView, this->getRefCnt());
+    }
+#endif
+
 private:
     GrVkImageView(VkImageView imageView) : INHERITED(), fImageView(imageView) {}
 
diff --git a/src/gpu/vk/GrVkPipeline.h b/src/gpu/vk/GrVkPipeline.h
index 0e818edadc..bec4aeb600 100644
--- a/src/gpu/vk/GrVkPipeline.h
+++ b/src/gpu/vk/GrVkPipeline.h
@@ -37,6 +37,11 @@ public:
 
     static void SetDynamicState(GrVkGpu*, GrVkCommandBuffer*, const GrPipeline&);
 
+#ifdef SK_TRACE_VK_RESOURCES
+    void dumpInfo() const override {
+        SkDebugf("GrVkPipeline: %d (%d refs)\n", fPipeline, this->getRefCnt());
+    }
+#endif
 
 private:
     GrVkPipeline(VkPipeline pipeline) : INHERITED(), fPipeline(pipeline) {}
diff --git a/src/gpu/vk/GrVkRenderPass.h b/src/gpu/vk/GrVkRenderPass.h
index 997989d073..5438a10dfc 100644
--- a/src/gpu/vk/GrVkRenderPass.h
+++ b/src/gpu/vk/GrVkRenderPass.h
@@ -120,6 +120,12 @@ public:
 
     void genKey(GrProcessorKeyBuilder* b) const;
 
+#ifdef SK_TRACE_VK_RESOURCES
+    void dumpInfo() const override {
+        SkDebugf("GrVkRenderPass: %d (%d refs)\n", fRenderPass, this->getRefCnt());
+    }
+#endif
+
 private:
     GrVkRenderPass(const GrVkRenderPass&);
 
diff --git a/src/gpu/vk/GrVkResource.h b/src/gpu/vk/GrVkResource.h
index 6c08f4ee55..d42ec70d76 100644
--- a/src/gpu/vk/GrVkResource.h
+++ b/src/gpu/vk/GrVkResource.h
@@ -15,9 +15,9 @@
 class GrVkGpu;
 
 // uncomment to enable tracing of resource refs
-//#ifdef SK_DEBUG
-//#define SK_TRACE_VK_RESOURCES
-//#endif
+#ifdef SK_DEBUG
+#define SK_TRACE_VK_RESOURCES
+#endif
 
 /** \class GrVkResource
 
@@ -121,6 +121,12 @@ public:
     }
 #endif
 
+#ifdef SK_TRACE_VK_RESOURCES
+    /** Output a human-readable dump of this resource's information
+     */
+    virtual void dumpInfo() const = 0;
+#endif
+
 private:
     /** Must be implemented by any subclasses.
      *  Deletes any Vk data associated with this resource
diff --git a/src/gpu/vk/GrVkResourceProvider.cpp b/src/gpu/vk/GrVkResourceProvider.cpp
index 4f6c7927c7..19a2f7d87f 100644
--- a/src/gpu/vk/GrVkResourceProvider.cpp
+++ b/src/gpu/vk/GrVkResourceProvider.cpp
@@ -221,8 +221,16 @@ void GrVkResourceProvider::getUniformDescriptorSet(VkDescriptorSet* ds,
     *outPool = fUniformDescPool;
 }
 
-GrVkPrimaryCommandBuffer* GrVkResourceProvider::createPrimaryCommandBuffer() {
-    GrVkPrimaryCommandBuffer* cmdBuffer = GrVkPrimaryCommandBuffer::Create(fGpu, fGpu->cmdPool());
+GrVkPrimaryCommandBuffer* GrVkResourceProvider::findOrCreatePrimaryCommandBuffer() {
+    GrVkPrimaryCommandBuffer* cmdBuffer = nullptr;
+    int count = fAvailableCommandBuffers.count();
+    if (count > 0) {
+        cmdBuffer = fAvailableCommandBuffers[count -1];
+        SkASSERT(cmdBuffer->finished(fGpu));
+        fAvailableCommandBuffers.removeShuffle(count - 1);
+    } else {
+        cmdBuffer = GrVkPrimaryCommandBuffer::Create(fGpu, fGpu->cmdPool());
+    }
     fActiveCommandBuffers.push_back(cmdBuffer);
     cmdBuffer->ref();
     return cmdBuffer;
@@ -231,20 +239,53 @@ GrVkPrimaryCommandBuffer* GrVkResourceProvider::createPrimaryCommandBuffer() {
 void GrVkResourceProvider::checkCommandBuffers() {
     for (int i = fActiveCommandBuffers.count()-1; i >= 0; --i) {
         if (fActiveCommandBuffers[i]->finished(fGpu)) {
-            fActiveCommandBuffers[i]->unref(fGpu);
+            GrVkPrimaryCommandBuffer* cmdBuffer = fActiveCommandBuffers[i];
+            cmdBuffer->reset(fGpu);
+            fAvailableCommandBuffers.push_back(cmdBuffer);
             fActiveCommandBuffers.removeShuffle(i);
         }
     }
 }
 
+GrVkSecondaryCommandBuffer* GrVkResourceProvider::findOrCreateSecondaryCommandBuffer() {
+    GrVkSecondaryCommandBuffer* cmdBuffer = nullptr;
+    int count = fAvailableSecondaryCommandBuffers.count();
+    if (count > 0) {
+        cmdBuffer = fAvailableSecondaryCommandBuffers[count-1];
+        fAvailableSecondaryCommandBuffers.removeShuffle(count - 1);
+    } else {
+        cmdBuffer = GrVkSecondaryCommandBuffer::Create(fGpu, fGpu->cmdPool());
+    }
+    return cmdBuffer;
+}
+
+void GrVkResourceProvider::recycleSecondaryCommandBuffer(GrVkSecondaryCommandBuffer* cb) {
+    cb->reset(fGpu);
+    fAvailableSecondaryCommandBuffers.push_back(cb);
+}
+
 void GrVkResourceProvider::destroyResources() {
-    // release our current command buffers
+    // release our active command buffers
     for (int i = 0; i < fActiveCommandBuffers.count(); ++i) {
         SkASSERT(fActiveCommandBuffers[i]->finished(fGpu));
         SkASSERT(fActiveCommandBuffers[i]->unique());
         fActiveCommandBuffers[i]->unref(fGpu);
     }
     fActiveCommandBuffers.reset();
+    // release our available command buffers
+    for (int i = 0; i < fAvailableCommandBuffers.count(); ++i) {
+        SkASSERT(fAvailableCommandBuffers[i]->finished(fGpu));
+        SkASSERT(fAvailableCommandBuffers[i]->unique());
+        fAvailableCommandBuffers[i]->unref(fGpu);
+    }
+    fAvailableCommandBuffers.reset();
+
+    // release our available secondary command buffers
+    for (int i = 0; i < fAvailableSecondaryCommandBuffers.count(); ++i) {
+        SkASSERT(fAvailableSecondaryCommandBuffers[i]->unique());
+        fAvailableSecondaryCommandBuffers[i]->unref(fGpu);
+    }
+    fAvailableSecondaryCommandBuffers.reset();
 
     // loop over all render pass sets to make sure we destroy all the internal VkRenderPasses
     for (int i = 0; i < fRenderPassArray.count(); ++i) {
@@ -273,17 +314,38 @@ void GrVkResourceProvider::destroyResources() {
     fUniformDescPool->unref(fGpu);
 
 #ifdef SK_TRACE_VK_RESOURCES
+    if (GrVkResource::fTrace.count()) {
+        SkTDynamicHash<GrVkResource, uint32_t>::Iter iter(&GrVkResource::fTrace);
+        for (; !iter.done(); ++iter) {
+            (*iter).dumpInfo();
+        }
+    }
     SkASSERT(0 == GrVkResource::fTrace.count());
 #endif
 }
 
 void GrVkResourceProvider::abandonResources() {
-    // release our current command buffers
+    // release our active command buffers
     for (int i = 0; i < fActiveCommandBuffers.count(); ++i) {
         SkASSERT(fActiveCommandBuffers[i]->finished(fGpu));
+        SkASSERT(fActiveCommandBuffers[i]->unique());
         fActiveCommandBuffers[i]->unrefAndAbandon();
     }
     fActiveCommandBuffers.reset();
+    // release our available command buffers
+    for (int i = 0; i < fAvailableCommandBuffers.count(); ++i) {
+        SkASSERT(fAvailableCommandBuffers[i]->finished(fGpu));
+        SkASSERT(fAvailableCommandBuffers[i]->unique());
+        fAvailableCommandBuffers[i]->unrefAndAbandon();
+    }
+    fAvailableCommandBuffers.reset();
+
+    // release our available secondary command buffers
+    for (int i = 0; i < fAvailableSecondaryCommandBuffers.count(); ++i) {
+        SkASSERT(fAvailableSecondaryCommandBuffers[i]->unique());
+        fAvailableSecondaryCommandBuffers[i]->unrefAndAbandon();
+    }
+    fAvailableSecondaryCommandBuffers.reset();
 
     // loop over all render pass sets to make sure we destroy all the internal VkRenderPasses
     for (int i = 0; i < fRenderPassArray.count(); ++i) {
@@ -306,6 +368,12 @@ void GrVkResourceProvider::abandonResources() {
     fUniformDescPool->unrefAndAbandon();
 
 #ifdef SK_TRACE_VK_RESOURCES
+    if (GrVkResource::fTrace.count()) {
+        SkTDynamicHash<GrVkResource, uint32_t>::Iter iter(&GrVkResource::fTrace);
+        for (; !iter.done(); ++iter) {
+            (*iter).dumpInfo();
+        }
+    }
     SkASSERT(0 == GrVkResource::fTrace.count());
 #endif
 }
diff --git a/src/gpu/vk/GrVkResourceProvider.h b/src/gpu/vk/GrVkResourceProvider.h
index d4383af96a..3891539b0c 100644
--- a/src/gpu/vk/GrVkResourceProvider.h
+++ b/src/gpu/vk/GrVkResourceProvider.h
@@ -25,11 +25,12 @@
 class GrPipeline;
 class GrPrimitiveProcessor;
 class GrTextureParams;
-class GrVkPrimaryCommandBuffer;
 class GrVkGpu;
 class GrVkPipeline;
+class GrVkPrimaryCommandBuffer;
 class GrVkRenderTarget;
 class GrVkSampler;
+class GrVkSecondaryCommandBuffer;
 
 class GrVkResourceProvider {
 public:
@@ -76,10 +77,12 @@ public:
                                          const GrVkRenderPass::LoadStoreOps& resolveOps,
                                          const GrVkRenderPass::LoadStoreOps& stencilOps);
 
-
-    GrVkPrimaryCommandBuffer* createPrimaryCommandBuffer();
+    GrVkPrimaryCommandBuffer* findOrCreatePrimaryCommandBuffer();
     void checkCommandBuffers();
 
+    GrVkSecondaryCommandBuffer* findOrCreateSecondaryCommandBuffer();
+    void recycleSecondaryCommandBuffer(GrVkSecondaryCommandBuffer* cb);
+
     // Finds or creates a compatible GrVkDescriptorPool for the requested type and count.
     // The refcount is incremented and a pointer returned.
     // TODO: Currently this will just create a descriptor pool without holding onto a ref itself
@@ -200,8 +203,13 @@ private:
 
     SkSTArray<4, CompatibleRenderPassSet> fRenderPassArray;
 
-    // Array of CommandBuffers that are currently in flight
+    // Array of PrimaryCommandBuffers that are currently in flight
     SkSTArray<4, GrVkPrimaryCommandBuffer*> fActiveCommandBuffers;
+    // Array of available primary command buffers that are not in flight
+    SkSTArray<4, GrVkPrimaryCommandBuffer*> fAvailableCommandBuffers;
+
+    // Array of available secondary command buffers
+    SkSTArray<16, GrVkSecondaryCommandBuffer*> fAvailableSecondaryCommandBuffers;
 
     // Stores GrVkSampler objects that we've already created so we can reuse them across multiple
     // GrVkPipelineStates
diff --git a/src/gpu/vk/GrVkSampler.h b/src/gpu/vk/GrVkSampler.h
index 2aa5017aad..c0f60e4217 100644
--- a/src/gpu/vk/GrVkSampler.h
+++ b/src/gpu/vk/GrVkSampler.h
@@ -28,6 +28,13 @@ public:
 
     static const uint16_t& GetKey(const GrVkSampler& sampler) { return sampler.fKey; }
     static uint32_t Hash(const uint16_t& key) { return key; }
+
+#ifdef SK_TRACE_VK_RESOURCES
+    void dumpInfo() const override {
+        SkDebugf("GrVkSampler: %d (%d refs)\n", fSampler, this->getRefCnt());
+    }
+#endif
+
 private:
     GrVkSampler(VkSampler sampler, uint16_t key) : INHERITED(), fSampler(sampler), fKey(key) {}