Vulkan:Integrate SecondaryCommandBuffers

Integrate the custom SecondaryCommandBuffer type into the CommandGraph
nodes by adding new ANGLE_USE_CUSTOM_VULKAN_CMD_BUFFERS define that can
be set in the BUILD gn args with angle_enable_custom_vulkan_cmd_buffers
set to "true."
Initially the custom cmd buffers are disabled by default.

This adds some support functions to SecondaryCommandBuffer to make the
integration easier by matching the wrapped cmd buffer interface:
initialize(), end(), valid().

Bug: angleproject:3136
Change-Id: Ib910554583192550757bb8ce89914e3ea8737988
Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/1526556
Commit-Queue: Tobin Ehlis <tobine@google.com>
Reviewed-by: Jamie Madill <jmadill@chromium.org>
diff --git a/src/libANGLE/renderer/vulkan/CommandGraph.cpp b/src/libANGLE/renderer/vulkan/CommandGraph.cpp
index cd24f0e..4fb85c6 100644
--- a/src/libANGLE/renderer/vulkan/CommandGraph.cpp
+++ b/src/libANGLE/renderer/vulkan/CommandGraph.cpp
@@ -25,14 +25,28 @@
 {
 namespace
 {
+ANGLE_MAYBE_UNUSED
 angle::Result InitAndBeginCommandBuffer(vk::Context *context,
                                         const CommandPool &commandPool,
                                         const VkCommandBufferInheritanceInfo &inheritanceInfo,
                                         VkCommandBufferUsageFlags flags,
+                                        angle::PoolAllocator *poolAllocator,
+                                        SecondaryCommandBuffer *commandBuffer)
+{
+    ASSERT(!commandBuffer->valid());
+    commandBuffer->initialize(poolAllocator);
+    return angle::Result::Continue;
+}
+
+ANGLE_MAYBE_UNUSED
+angle::Result InitAndBeginCommandBuffer(vk::Context *context,
+                                        const CommandPool &commandPool,
+                                        const VkCommandBufferInheritanceInfo &inheritanceInfo,
+                                        VkCommandBufferUsageFlags flags,
+                                        angle::PoolAllocator *poolAllocator,
                                         CommandBuffer *commandBuffer)
 {
     ASSERT(!commandBuffer->valid());
-
     VkCommandBufferAllocateInfo createInfo = {};
     createInfo.sType                       = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
     createInfo.commandPool                 = commandPool.getHandle();
@@ -124,6 +138,26 @@
     kLabelColors[colorIndex].writeData(label->color);
 }
 
+#if ANGLE_USE_CUSTOM_VULKAN_CMD_BUFFERS
+static constexpr VkSubpassContents kRenderPassContents = VK_SUBPASS_CONTENTS_INLINE;
+#else
+static constexpr VkSubpassContents kRenderPassContents =
+    VK_SUBPASS_CONTENTS_SECONDARY_COMMAND_BUFFERS;
+#endif
+
+// Helpers to unify executeCommands call based on underlying cmd buffer type
+ANGLE_MAYBE_UNUSED
+void ExecuteCommands(CommandBuffer *primCmdBuffer, SecondaryCommandBuffer *secCmdBuffer)
+{
+    secCmdBuffer->executeCommands(primCmdBuffer->getHandle());
+}
+
+ANGLE_MAYBE_UNUSED
+void ExecuteCommands(CommandBuffer *primCmdBuffer, CommandBuffer *secCmdBuffer)
+{
+    primCmdBuffer->executeCommands(1, secCmdBuffer);
+}
+
 }  // anonymous namespace
 
 // CommandGraphResource implementation.
@@ -139,7 +173,7 @@
 }
 
 angle::Result CommandGraphResource::recordCommands(Context *context,
-                                                   CommandBuffer **commandBufferOut)
+                                                   CommandBufferT **commandBufferOut)
 {
     updateQueueSerial(context->getRenderer()->getCurrentQueueSerial());
 
@@ -150,7 +184,7 @@
             context, context->getRenderer()->getCommandPool(), commandBufferOut);
     }
 
-    CommandBuffer *outsideRenderPassCommands = mCurrentWritingNode->getOutsideRenderPassCommands();
+    CommandBufferT *outsideRenderPassCommands = mCurrentWritingNode->getOutsideRenderPassCommands();
     if (!outsideRenderPassCommands->valid())
     {
         ANGLE_TRY(mCurrentWritingNode->beginOutsideRenderPassRecording(
@@ -175,7 +209,7 @@
                                                     const gl::Rectangle &renderArea,
                                                     const RenderPassDesc &renderPassDesc,
                                                     const std::vector<VkClearValue> &clearValues,
-                                                    CommandBuffer **commandBufferOut)
+                                                    CommandBufferT **commandBufferOut)
 {
     // If a barrier has been inserted in the meantime, stop the command buffer.
     if (!hasChildlessWritingNode())
@@ -251,9 +285,11 @@
 }
 
 // CommandGraphNode implementation.
-CommandGraphNode::CommandGraphNode(CommandGraphNodeFunction function)
+CommandGraphNode::CommandGraphNode(CommandGraphNodeFunction function,
+                                   angle::PoolAllocator *poolAllocator)
     : mRenderPassClearValues{},
       mFunction(function),
+      mPoolAllocator(poolAllocator),
       mQueryPool(VK_NULL_HANDLE),
       mQueryIndex(0),
       mFenceSyncEvent(VK_NULL_HANDLE),
@@ -273,15 +309,9 @@
     mInsideRenderPassCommands.releaseHandle();
 }
 
-CommandBuffer *CommandGraphNode::getOutsideRenderPassCommands()
-{
-    ASSERT(!mHasChildren);
-    return &mOutsideRenderPassCommands;
-}
-
 angle::Result CommandGraphNode::beginOutsideRenderPassRecording(Context *context,
                                                                 const CommandPool &commandPool,
-                                                                CommandBuffer **commandsOut)
+                                                                CommandBufferT **commandsOut)
 {
     ASSERT(!mHasChildren);
 
@@ -295,7 +325,7 @@
     inheritanceInfo.queryFlags         = 0;
     inheritanceInfo.pipelineStatistics = 0;
 
-    ANGLE_TRY(InitAndBeginCommandBuffer(context, commandPool, inheritanceInfo, 0,
+    ANGLE_TRY(InitAndBeginCommandBuffer(context, commandPool, inheritanceInfo, 0, mPoolAllocator,
                                         &mOutsideRenderPassCommands));
 
     *commandsOut = &mOutsideRenderPassCommands;
@@ -303,7 +333,7 @@
 }
 
 angle::Result CommandGraphNode::beginInsideRenderPassRecording(Context *context,
-                                                               CommandBuffer **commandsOut)
+                                                               CommandBufferT **commandsOut)
 {
     ASSERT(!mHasChildren);
 
@@ -323,9 +353,10 @@
     inheritanceInfo.queryFlags         = 0;
     inheritanceInfo.pipelineStatistics = 0;
 
-    ANGLE_TRY(InitAndBeginCommandBuffer(
-        context, context->getRenderer()->getCommandPool(), inheritanceInfo,
-        VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT, &mInsideRenderPassCommands));
+    ANGLE_TRY(InitAndBeginCommandBuffer(context, context->getRenderer()->getCommandPool(),
+                                        inheritanceInfo,
+                                        VK_COMMAND_BUFFER_USAGE_RENDER_PASS_CONTINUE_BIT,
+                                        mPoolAllocator, &mInsideRenderPassCommands));
 
     *commandsOut = &mInsideRenderPassCommands;
     return angle::Result::Continue;
@@ -462,7 +493,7 @@
             if (mOutsideRenderPassCommands.valid())
             {
                 ANGLE_VK_TRY(context, mOutsideRenderPassCommands.end());
-                primaryCommandBuffer->executeCommands(1, &mOutsideRenderPassCommands);
+                ExecuteCommands(primaryCommandBuffer, &mOutsideRenderPassCommands);
             }
 
             if (mInsideRenderPassCommands.valid())
@@ -488,9 +519,8 @@
                 beginInfo.clearValueCount = mRenderPassDesc.attachmentCount();
                 beginInfo.pClearValues    = mRenderPassClearValues.data();
 
-                primaryCommandBuffer->beginRenderPass(
-                    beginInfo, VK_SUBPASS_CONTENTS_SECONDARY_COMMAND_BUFFERS);
-                primaryCommandBuffer->executeCommands(1, &mInsideRenderPassCommands);
+                primaryCommandBuffer->beginRenderPass(beginInfo, kRenderPassContents);
+                ExecuteCommands(primaryCommandBuffer, &mInsideRenderPassCommands);
                 primaryCommandBuffer->endRenderPass();
             }
             break;
@@ -600,8 +630,10 @@
 }
 
 // CommandGraph implementation.
-CommandGraph::CommandGraph(bool enableGraphDiagnostics)
-    : mEnableGraphDiagnostics(enableGraphDiagnostics), mLastBarrierIndex(kInvalidNodeIndex)
+CommandGraph::CommandGraph(bool enableGraphDiagnostics, angle::PoolAllocator *poolAllocator)
+    : mEnableGraphDiagnostics(enableGraphDiagnostics),
+      mPoolAllocator(poolAllocator),
+      mLastBarrierIndex(kInvalidNodeIndex)
 {}
 
 CommandGraph::~CommandGraph()
@@ -612,7 +644,7 @@
 CommandGraphNode *CommandGraph::allocateNode(CommandGraphNodeFunction function)
 {
     // TODO(jmadill): Use a pool allocator for the CPU node allocations.
-    CommandGraphNode *newCommands = new CommandGraphNode(function);
+    CommandGraphNode *newCommands = new CommandGraphNode(function, mPoolAllocator);
     mNodes.emplace_back(newCommands);
     return newCommands;
 }