Enabled deferred contexts in Vk backend

5 files changed, 86 insertions, 55 deletions

diff --git a/Graphics/GraphicsEngineVulkan/include/CommandListVkImpl.h b/Graphics/GraphicsEngineVulkan/include/CommandListVkImpl.h
index 645218dc..0ed4b97e 100644
--- a/Graphics/GraphicsEngineVulkan/include/CommandListVkImpl.h
+++ b/Graphics/GraphicsEngineVulkan/include/CommandListVkImpl.h
@@ -26,6 +26,7 @@
 /// \file
 /// Declaration of Diligent::CommandListVkImpl class
 
+#include "vulkan.h"
 #include "CommandListBase.h"
 
 namespace Diligent
@@ -36,26 +37,28 @@ class CommandListVkImpl : public CommandListBase<ICommandList>
 {
 public:
     typedef CommandListBase<ICommandList> TCommandListBase;
-    CommandListVkImpl(IReferenceCounters *pRefCounters, IRenderDevice *pDevice, class CommandContext* pCmdContext) :
+    CommandListVkImpl(IReferenceCounters *pRefCounters, IRenderDevice *pDevice, IDeviceContext *pDeferredCtx, VkCommandBuffer vkCmdBuff) :
         TCommandListBase(pRefCounters, pDevice),
-        m_pCmdContext(pCmdContext)
+        m_pDeferredCtx(pDeferredCtx),
+        m_vkCmdBuff(vkCmdBuff)
     {
     }
     
     ~CommandListVkImpl()
     {
-        VERIFY(m_pCmdContext == nullptr, "Destroying command list that was never executed");
+        VERIFY(m_vkCmdBuff == VK_NULL_HANDLE && !m_pDeferredCtx, "Destroying command list that was never executed");
     }
 
-    CommandContext* Close()
+    void Close(VkCommandBuffer& CmdBuff, RefCntAutoPtr<IDeviceContext>& pDeferredCtx)
     {
-        CommandContext* pCmdContext = m_pCmdContext;
-        m_pCmdContext = nullptr;
-        return pCmdContext;
+        CmdBuff = m_vkCmdBuff;
+        m_vkCmdBuff = VK_NULL_HANDLE;
+        pDeferredCtx = std::move(m_pDeferredCtx);
     }
 
 private:
-    CommandContext* m_pCmdContext = nullptr;
+    RefCntAutoPtr<IDeviceContext> m_pDeferredCtx;
+    VkCommandBuffer m_vkCmdBuff;
 };
 
 }
diff --git a/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h b/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h
index 20f7c00e..a748ca58 100644
--- a/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h
+++ b/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h
@@ -144,7 +144,8 @@ private:
     void CommitScissorRects();
 
     inline void EnsureVkCmdBuffer();
-    inline void DisposeVkCmdBuffer();
+    inline void DisposeVkCmdBuffer(VkCommandBuffer vkCmdBuff);
+    inline void DisposeCurrentCmdBuffer();
     
     VulkanUtilities::VulkanCommandBuffer m_CommandBuffer;
 
@@ -162,16 +163,12 @@ private:
     
     
 #if 0
-    CComPtr<IVkCommandSignature> m_pDrawIndirectSignature;
-    CComPtr<IVkCommandSignature> m_pDrawIndexedIndirectSignature;
-    CComPtr<IVkCommandSignature> m_pDispatchIndirectSignature;
-    
     GenerateMipsHelper m_MipsGenerator;
 #endif
     class ShaderResourceCacheVk *m_pCommittedResourceCache = nullptr;
-#if 0
+
     FixedBlockMemoryAllocator m_CmdListAllocator;
-#endif
+
     const Uint32 m_ContextId;
 
     VulkanUtilities::VulkanCommandBufferPool m_CmdPool;
diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanCommandBufferPool.h b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanCommandBufferPool.h
index d898bae2..464b1e56 100644
--- a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanCommandBufferPool.h
+++ b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanCommandBufferPool.h
@@ -25,6 +25,7 @@
 
 #include <deque>
 #include <memory>
+#include <mutex>
 #include "vulkan.h"
 #include "VulkanLogicalDevice.h"
 #include "VulkanObjectWrappers.h"
@@ -36,7 +37,8 @@ namespace VulkanUtilities
     public:
         VulkanCommandBufferPool(std::shared_ptr<const VulkanUtilities::VulkanLogicalDevice> LogicalDevice, 
                                 uint32_t queueFamilyIndex, 
-                                VkCommandPoolCreateFlags flags);
+                                VkCommandPoolCreateFlags flags,
+                                bool IsThreadSafe);
         VulkanCommandBufferPool(const VulkanCommandBufferPool&) = delete;
         VulkanCommandBufferPool(VulkanCommandBufferPool&&) = delete;
         VulkanCommandBufferPool& operator = (const VulkanCommandBufferPool&) = delete;
@@ -52,7 +54,10 @@ namespace VulkanUtilities
         // Shared point to logical device must be defined before the command pool
         std::shared_ptr<const VulkanUtilities::VulkanLogicalDevice> m_LogicalDevice;
         CommandPoolWrapper m_CmdPool;
+        
+        const bool m_IsThreadSafe;
 
+        std::mutex m_Mutex;
         // fist    - the fence value associated with the command buffer when it was executed
         // second  - the command buffer
         typedef std::pair<uint64_t, VkCommandBuffer > QueueElemType;
diff --git a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
index 33d58923..9a27894f 100644
--- a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
@@ -38,10 +38,11 @@ namespace Diligent
     DeviceContextVkImpl::DeviceContextVkImpl( IReferenceCounters *pRefCounters, RenderDeviceVkImpl *pDeviceVkImpl, bool bIsDeferred, const EngineVkAttribs &Attribs, Uint32 ContextId) :
         TDeviceContextBase(pRefCounters, pDeviceVkImpl, bIsDeferred),
         m_NumCommandsToFlush(bIsDeferred ? std::numeric_limits<decltype(m_NumCommandsToFlush)>::max() : Attribs.NumCommandsToFlushCmdBuffer),
-        /*m_MipsGenerator(pDeviceVkImpl->GetVkDevice()),
-        m_CmdListAllocator(GetRawAllocator(), sizeof(CommandListVkImpl), 64 ),*/
+        /*m_MipsGenerator(pDeviceVkImpl->GetVkDevice()),*/
+        m_CmdListAllocator(GetRawAllocator(), sizeof(CommandListVkImpl), 64 ),
         m_ContextId(ContextId),
-        m_CmdPool(pDeviceVkImpl->GetLogicalDevice().GetSharedPtr(), pDeviceVkImpl->GetCmdQueue()->GetQueueFamilyIndex(), VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT)
+        // Command pools for deferred contexts must be thread safe because finished command buffers are executed and released from another thread
+        m_CmdPool(pDeviceVkImpl->GetLogicalDevice().GetSharedPtr(), pDeviceVkImpl->GetCmdQueue()->GetQueueFamilyIndex(), VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, bIsDeferred)
     {
 #if 0
         auto *pVkDevice = pDeviceVkImpl->GetVkDevice();
@@ -73,7 +74,7 @@ namespace Diligent
     {
         if(m_bIsDeferred)
         {
-            DisposeVkCmdBuffer();
+            DisposeCurrentCmdBuffer();
         }
         else
         {
@@ -102,14 +103,20 @@ namespace Diligent
         }
     }
 
-    inline void DeviceContextVkImpl::DisposeVkCmdBuffer()
+    void DeviceContextVkImpl::DisposeVkCmdBuffer(VkCommandBuffer vkCmdBuff)
+    {
+        VERIFY_EXPR(vkCmdBuff != VK_NULL_HANDLE);
+        auto pDeviceVkImpl = m_pDevice.RawPtr<RenderDeviceVkImpl>();
+        m_CmdPool.DisposeCommandBuffer(vkCmdBuff, pDeviceVkImpl->GetNextFenceValue());
+    }
+
+    inline void DeviceContextVkImpl::DisposeCurrentCmdBuffer()
     {
         VERIFY(m_CommandBuffer.GetState().RenderPass == VK_NULL_HANDLE, "Disposing command buffer with unifinished render pass");
         auto vkCmdBuff = m_CommandBuffer.GetVkCmdBuffer();
         if(vkCmdBuff != VK_NULL_HANDLE)
         {
-            auto pDeviceVkImpl = m_pDevice.RawPtr<RenderDeviceVkImpl>();
-            m_CmdPool.DisposeCommandBuffer(vkCmdBuff, pDeviceVkImpl->GetNextFenceValue());
+            DisposeVkCmdBuffer(vkCmdBuff);
             m_CommandBuffer.Reset();
         }
     }
@@ -762,7 +769,7 @@ namespace Diligent
 
         if (vkCmdBuff != VK_NULL_HANDLE)
         {
-            DisposeVkCmdBuffer();
+            DisposeCurrentCmdBuffer();
         }
 
         m_State = ContextState{};
@@ -1044,19 +1051,19 @@ namespace Diligent
 
     void DeviceContextVkImpl::FinishCommandList(class ICommandList **ppCommandList)
     {
-#if 0
+        auto vkCmdBuff = m_CommandBuffer.GetVkCmdBuffer();
         CommandListVkImpl *pCmdListVk( NEW_RC_OBJ(m_CmdListAllocator, "CommandListVkImpl instance", CommandListVkImpl)
-                                                       (m_pDevice, m_pCurrCmdCtx) );
+                                                 (m_pDevice, this, vkCmdBuff) );
         pCmdListVk->QueryInterface( IID_CommandList, reinterpret_cast<IObject**>(ppCommandList) );
-        m_pCurrCmdCtx = nullptr;
-        //Flush(); 
+        
+        m_CommandBuffer.SetVkCmdBuffer(VK_NULL_HANDLE);
+        //Flush();
 
         m_CommandBuffer.Reset();
         m_State = ContextState{};
         m_pPipelineState.Release();
 
         InvalidateState();
-#endif
     }
 
     void DeviceContextVkImpl::ExecuteCommandList(class ICommandList *pCommandList)
@@ -1066,15 +1073,22 @@ namespace Diligent
             LOG_ERROR("Only immediate context can execute command list");
             return;
         }
-#if 0
+
         // First execute commands in this context
         Flush();
 
         InvalidateState();
 
         CommandListVkImpl* pCmdListVk = ValidatedCast<CommandListVkImpl>(pCommandList);
-        m_pDevice.RawPtr<RenderDeviceVkImpl>()->CloseAndExecuteCommandContext(pCmdListVk->Close(), true);
-#endif
+        VkCommandBuffer vkCmdBuff = VK_NULL_HANDLE;
+        RefCntAutoPtr<IDeviceContext> pDeferredCtx;
+        pCmdListVk->Close(vkCmdBuff, pDeferredCtx);
+        VERIFY(vkCmdBuff != VK_NULL_HANDLE, "Trying to execute empty command buffer");
+        VERIFY_EXPR(pDeferredCtx);
+        m_pDevice.RawPtr<RenderDeviceVkImpl>()->ExecuteCommandBuffer(vkCmdBuff, true);
+        // It is OK to dispose command buffer from another thread. We are not going to
+        // record any commands and only need to add the buffer to the queue
+        pDeferredCtx.RawPtr<DeviceContextVkImpl>()->DisposeVkCmdBuffer(vkCmdBuff);
     }
 
     void DeviceContextVkImpl::TransitionImageLayout(ITexture *pTexture, VkImageLayout NewLayout)
diff --git a/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanCommandBufferPool.cpp b/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanCommandBufferPool.cpp
index b5ec40b0..bf120f88 100644
--- a/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanCommandBufferPool.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanCommandBufferPool.cpp
@@ -32,8 +32,10 @@ namespace VulkanUtilities
 {
     VulkanCommandBufferPool::VulkanCommandBufferPool(std::shared_ptr<const VulkanUtilities::VulkanLogicalDevice> LogicalDevice,
                                                      uint32_t queueFamilyIndex, 
-                                                     VkCommandPoolCreateFlags flags) :
-        m_LogicalDevice(LogicalDevice)
+                                                     VkCommandPoolCreateFlags flags,
+                                                     bool IsThreadSafe) :
+        m_LogicalDevice(LogicalDevice),
+        m_IsThreadSafe(IsThreadSafe)
     {
         VkCommandPoolCreateInfo CmdPoolCI = {};
         CmdPoolCI.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
@@ -53,30 +55,36 @@ namespace VulkanUtilities
     {
         VkCommandBuffer CmdBuffer = VK_NULL_HANDLE;
 
-        if (!m_DiscardedCmdBuffers.empty())
         {
-            // Pick the oldest cmd buffer at the front of the deque
-            // If this buffer is not yet available, there is no point in
-            // looking at other buffers since they were released even
-            // later
-            auto& OldestBuff = m_DiscardedCmdBuffers.front();
+            std::unique_lock<std::mutex> Lock(m_Mutex, std::defer_lock);
+            if (m_IsThreadSafe)
+                Lock.lock();
 
-            // Note that LastCompletedFence only grows. So if after we queried
-            // the value, the actual value is increased in other thread, this will not
-            // be an issue as the only consequence is that potentially available  
-            // cmd buffer may not be used.
-
-            // OldestBuff.first is the fence value that was signaled AFTER the
-            // command buffer has been submitted. If LastCompletedFence is at least
-            // this value, the buffer can be safely reused
-            if (LastCompletedFence >= OldestBuff.first)
+            if (!m_DiscardedCmdBuffers.empty())
             {
-                CmdBuffer = OldestBuff.second;
-                auto err = vkResetCommandBuffer(CmdBuffer, 
-                    0 // VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT -  specifies that most or all memory resources currently owned by the command buffer should be returned to the parent command pool.
-                );
-                VERIFY(err == VK_SUCCESS, "Failed to reset command buffer");
-                m_DiscardedCmdBuffers.pop_front();
+                // Pick the oldest cmd buffer at the front of the deque
+                // If this buffer is not yet available, there is no point in
+                // looking at other buffers since they were released even
+                // later
+                auto& OldestBuff = m_DiscardedCmdBuffers.front();
+
+                // Note that LastCompletedFence only grows. So if after we queried
+                // the value, the actual value is increased in other thread, this will not
+                // be an issue as the only consequence is that potentially available  
+                // cmd buffer may not be used.
+
+                // OldestBuff.first is the fence value that was signaled AFTER the
+                // command buffer has been submitted. If LastCompletedFence is at least
+                // this value, the buffer can be safely reused
+                if (LastCompletedFence >= OldestBuff.first)
+                {
+                    CmdBuffer = OldestBuff.second;
+                    auto err = vkResetCommandBuffer(CmdBuffer, 
+                        0 // VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT -  specifies that most or all memory resources currently owned by the command buffer should be returned to the parent command pool.
+                    );
+                    VERIFY(err == VK_SUCCESS, "Failed to reset command buffer");
+                    m_DiscardedCmdBuffers.pop_front();
+                }
             }
         }
 
@@ -108,6 +116,10 @@ namespace VulkanUtilities
 
     void VulkanCommandBufferPool::DisposeCommandBuffer(VkCommandBuffer CmdBuffer, uint64_t FenceValue)
     {
+        std::unique_lock<std::mutex> Lock(m_Mutex, std::defer_lock);
+        if (m_IsThreadSafe)
+            Lock.lock();
+
         // FenceValue is the value that was signaled by the command queue after it 
         // executed the command buffer
         m_DiscardedCmdBuffers.emplace_back(FenceValue, CmdBuffer);