Fixed Vulkan command buffer leak

9 files changed, 143 insertions, 126 deletions

diff --git a/Graphics/GraphicsEngineVulkan/include/CommandPoolManager.hpp b/Graphics/GraphicsEngineVulkan/include/CommandPoolManager.hpp
index f66181b7..8bb41583 100644
--- a/Graphics/GraphicsEngineVulkan/include/CommandPoolManager.hpp
+++ b/Graphics/GraphicsEngineVulkan/include/CommandPoolManager.hpp
@@ -32,19 +32,18 @@
 #include <atomic>
 #include "STDAllocator.hpp"
 #include "VulkanUtilities/VulkanObjectWrappers.hpp"
+#include "VulkanUtilities/VulkanLogicalDevice.hpp"
 
 namespace Diligent
 {
 
-class RenderDeviceVkImpl;
-
 class CommandPoolManager
 {
 public:
-    CommandPoolManager(RenderDeviceVkImpl&      DeviceVkImpl,
-                       std::string              Name,
-                       uint32_t                 queueFamilyIndex,
-                       VkCommandPoolCreateFlags flags) noexcept;
+    CommandPoolManager(const VulkanUtilities::VulkanLogicalDevice& LogicalDevice,
+                       std::string                                 Name,
+                       uint32_t                                    queueFamilyIndex,
+                       VkCommandPoolCreateFlags                    flags) noexcept;
 
     // clang-format off
     CommandPoolManager             (const CommandPoolManager&)  = delete;
@@ -58,8 +57,6 @@ public:
     // Allocates Vulkan command pool.
     VulkanUtilities::CommandPoolWrapper AllocateCommandPool(const char* DebugName = nullptr);
 
-    void SafeReleaseCommandPool(VulkanUtilities::CommandPoolWrapper&& CmdPool, Uint32 CmdQueueIndex, Uint64 FenceValue);
-
     void DestroyPools();
 
 #ifdef DILIGENT_DEVELOPMENT
@@ -69,11 +66,12 @@ public:
     }
 #endif
 
-private:
     // Returns command pool to the list of available pools. The GPU must have finished using the pool
-    void FreeCommandPool(VulkanUtilities::CommandPoolWrapper&& CmdPool);
+    void RecycleCommandPool(VulkanUtilities::CommandPoolWrapper&& CmdPool);
+
+private:
+    const VulkanUtilities::VulkanLogicalDevice& m_LogicalDevice;
 
-    RenderDeviceVkImpl&            m_DeviceVkImpl;
     const std::string              m_Name;
     const uint32_t                 m_QueueFamilyIndex;
     const VkCommandPoolCreateFlags m_CmdPoolFlags;
@@ -82,7 +80,7 @@ private:
     std::deque<VulkanUtilities::CommandPoolWrapper, STDAllocatorRawMem<VulkanUtilities::CommandPoolWrapper>> m_CmdPools;
 
 #ifdef DILIGENT_DEVELOPMENT
-    std::atomic_int32_t m_AllocatedPoolCounter;
+    std::atomic_int32_t m_AllocatedPoolCounter{0};
 #endif
 };
 
diff --git a/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.hpp b/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.hpp
index b3d1e1b2..d5de333c 100644
--- a/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.hpp
+++ b/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.hpp
@@ -426,7 +426,7 @@ private:
         m_State.NumCommands = m_State.NumCommands != 0 ? m_State.NumCommands : 1;
         if (m_CommandBuffer.GetVkCmdBuffer() == VK_NULL_HANDLE)
         {
-            auto vkCmdBuff = m_CmdPool.GetCommandBuffer();
+            auto vkCmdBuff = m_CmdPool->GetCommandBuffer();
             m_CommandBuffer.SetVkCmdBuffer(vkCmdBuff);
         }
     }
@@ -600,10 +600,11 @@ private:
     };
     std::unordered_map<MappedTextureKey, MappedTexture, MappedTextureKey::Hasher> m_MappedTextures;
 
-    VulkanUtilities::VulkanCommandBufferPool m_CmdPool;
-    VulkanUploadHeap                         m_UploadHeap;
-    VulkanDynamicHeap                        m_DynamicHeap;
-    DynamicDescriptorSetAllocator            m_DynamicDescrSetAllocator;
+    std::unique_ptr<VulkanUtilities::VulkanCommandBufferPool> m_CmdPool;
+
+    VulkanUploadHeap              m_UploadHeap;
+    VulkanDynamicHeap             m_DynamicHeap;
+    DynamicDescriptorSetAllocator m_DynamicDescrSetAllocator;
 
     std::shared_ptr<GenerateMipsVkHelper> m_GenerateMipsHelper;
     RefCntAutoPtr<IShaderResourceBinding> m_GenerateMipsSRB;
diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanCommandBufferPool.hpp b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanCommandBufferPool.hpp
index bf3608a1..252a316c 100644
--- a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanCommandBufferPool.hpp
+++ b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanCommandBufferPool.hpp
@@ -56,9 +56,7 @@ public:
 
     VkCommandBuffer GetCommandBuffer(const char* DebugName = "");
     // The GPU must have finished with the command buffer being returned to the pool
-    void FreeCommandBuffer(VkCommandBuffer&& CmdBuffer);
-
-    CommandPoolWrapper&& Release();
+    void RecycleCommandBuffer(VkCommandBuffer&& CmdBuffer);
 
 #ifdef DILIGENT_DEVELOPMENT
     int32_t DvpGetBufferCounter() const
@@ -70,12 +68,13 @@ public:
 private:
     // Shared point to logical device must be defined before the command pool
     std::shared_ptr<const VulkanLogicalDevice> m_LogicalDevice;
-    CommandPoolWrapper                         m_CmdPool;
+
+    CommandPoolWrapper m_CmdPool;
 
     std::mutex                  m_Mutex;
     std::deque<VkCommandBuffer> m_CmdBuffers;
 #ifdef DILIGENT_DEVELOPMENT
-    std::atomic_int32_t m_BuffCounter;
+    std::atomic_int32_t m_BuffCounter{0};
 #endif
 };
 
diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanLogicalDevice.hpp b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanLogicalDevice.hpp
index 7de814e4..54660251 100644
--- a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanLogicalDevice.hpp
+++ b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanLogicalDevice.hpp
@@ -171,7 +171,8 @@ public:
     void ReleaseVulkanObject(AccelStructWrapper&&   AccelStruct) const;
 
     void FreeDescriptorSet(VkDescriptorPool Pool, VkDescriptorSet Set) const;
-
+    void FreeCommandBuffer(VkCommandPool Pool, VkCommandBuffer CmdBuffer) const;
+    
     VkMemoryRequirements GetBufferMemoryRequirements(VkBuffer vkBuffer) const;
     VkMemoryRequirements GetImageMemoryRequirements (VkImage  vkImage ) const;
     VkDeviceAddress      GetAccelerationStructureDeviceAddress(VkAccelerationStructureKHR AS) const;
diff --git a/Graphics/GraphicsEngineVulkan/src/CommandPoolManager.cpp b/Graphics/GraphicsEngineVulkan/src/CommandPoolManager.cpp
index 0175c2e1..27f5a25f 100644
--- a/Graphics/GraphicsEngineVulkan/src/CommandPoolManager.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/CommandPoolManager.cpp
@@ -32,36 +32,31 @@
 namespace Diligent
 {
 
-CommandPoolManager::CommandPoolManager(RenderDeviceVkImpl&      DeviceVkImpl,
-                                       std::string              Name,
-                                       uint32_t                 queueFamilyIndex,
-                                       VkCommandPoolCreateFlags flags) noexcept :
+CommandPoolManager::CommandPoolManager(const VulkanUtilities::VulkanLogicalDevice& LogicalDevice,
+                                       std::string                                 Name,
+                                       uint32_t                                    queueFamilyIndex,
+                                       VkCommandPoolCreateFlags                    flags) noexcept :
     // clang-format off
-    m_DeviceVkImpl    {DeviceVkImpl     },
+    m_LogicalDevice   {LogicalDevice    },
     m_Name            {std::move(Name)  },
     m_QueueFamilyIndex{queueFamilyIndex },
     m_CmdPoolFlags    {flags            },
     m_CmdPools        (STD_ALLOCATOR_RAW_MEM(VulkanUtilities::CommandPoolWrapper, GetRawAllocator(), "Allocator for deque<VulkanUtilities::CommandPoolWrapper>"))
 // clang-format on
 {
-#ifdef DILIGENT_DEVELOPMENT
-    m_AllocatedPoolCounter = 0;
-#endif
 }
 
 VulkanUtilities::CommandPoolWrapper CommandPoolManager::AllocateCommandPool(const char* DebugName)
 {
     std::lock_guard<std::mutex> LockGuard{m_Mutex};
 
-    const auto& LogicalDevice = m_DeviceVkImpl.GetLogicalDevice();
-
     VulkanUtilities::CommandPoolWrapper CmdPool;
     if (!m_CmdPools.empty())
     {
         CmdPool = std::move(m_CmdPools.front());
         m_CmdPools.pop_front();
 
-        LogicalDevice.ResetCommandPool(CmdPool);
+        m_LogicalDevice.ResetCommandPool(CmdPool);
     }
 
     if (CmdPool == VK_NULL_HANDLE)
@@ -73,7 +68,7 @@ VulkanUtilities::CommandPoolWrapper CommandPoolManager::AllocateCommandPool(cons
         CmdPoolCI.queueFamilyIndex = m_QueueFamilyIndex;
         CmdPoolCI.flags            = m_CmdPoolFlags;
 
-        CmdPool = LogicalDevice.CreateCommandPool(CmdPoolCI);
+        CmdPool = m_LogicalDevice.CreateCommandPool(CmdPoolCI);
         DEV_CHECK_ERR(CmdPool != VK_NULL_HANDLE, "Failed to create Vulkan command pool");
     }
 
@@ -83,54 +78,9 @@ VulkanUtilities::CommandPoolWrapper CommandPoolManager::AllocateCommandPool(cons
     return std::move(CmdPool);
 }
 
-void CommandPoolManager::SafeReleaseCommandPool(VulkanUtilities::CommandPoolWrapper&& CmdPool, Uint32 CmdQueueIndex, Uint64 FenceValue)
-{
-    class CommandPoolDeleter
-    {
-    public:
-        CommandPoolDeleter(CommandPoolManager& _CmdPoolMgr, VulkanUtilities::CommandPoolWrapper&& _Pool) :
-            // clang-format off
-            CmdPoolMgr{&_CmdPoolMgr    },
-            Pool      {std::move(_Pool)}
-        // clang-format on
-        {
-            VERIFY_EXPR(Pool != VK_NULL_HANDLE);
-        }
-
-        // clang-format off
-        CommandPoolDeleter             (const CommandPoolDeleter&)  = delete;
-        CommandPoolDeleter& operator = (const CommandPoolDeleter&)  = delete;
-        CommandPoolDeleter& operator = (      CommandPoolDeleter&&) = delete;
-
-        CommandPoolDeleter(CommandPoolDeleter&& rhs) : 
-            CmdPoolMgr{rhs.CmdPoolMgr     },
-            Pool      {std::move(rhs.Pool)}
-        {
-            rhs.CmdPoolMgr = nullptr;
-        }
-        // clang-format on
-
-        ~CommandPoolDeleter()
-        {
-            if (CmdPoolMgr != nullptr)
-            {
-                CmdPoolMgr->FreeCommandPool(std::move(Pool));
-            }
-        }
-
-    private:
-        CommandPoolManager*                 CmdPoolMgr;
-        VulkanUtilities::CommandPoolWrapper Pool;
-    };
-
-    // Discard command pool directly to the release queue since we know exactly which queue it was submitted to
-    // as well as the associated FenceValue
-    m_DeviceVkImpl.GetReleaseQueue(CmdQueueIndex).DiscardResource(CommandPoolDeleter{*this, std::move(CmdPool)}, FenceValue);
-}
-
-void CommandPoolManager::FreeCommandPool(VulkanUtilities::CommandPoolWrapper&& CmdPool)
+void CommandPoolManager::RecycleCommandPool(VulkanUtilities::CommandPoolWrapper&& CmdPool)
 {
-    std::lock_guard<std::mutex> LockGuard(m_Mutex);
+    std::lock_guard<std::mutex> LockGuard{m_Mutex};
 #ifdef DILIGENT_DEVELOPMENT
     --m_AllocatedPoolCounter;
 #endif
@@ -139,8 +89,8 @@ void CommandPoolManager::FreeCommandPool(VulkanUtilities::CommandPoolWrapper&& C
 
 void CommandPoolManager::DestroyPools()
 {
-    std::lock_guard<std::mutex> LockGuard(m_Mutex);
-    DEV_CHECK_ERR(m_AllocatedPoolCounter == 0, m_AllocatedPoolCounter, " pool(s) have not been freed. This will cause a crash if the references to these pools are still in release queues when CommandPoolManager::FreeCommandPool() is called for destroyed CommandPoolManager object.");
+    std::lock_guard<std::mutex> LockGuard{m_Mutex};
+    DEV_CHECK_ERR(m_AllocatedPoolCounter == 0, m_AllocatedPoolCounter, " pool(s) have not been recycled. This will cause a crash if the references to these pools are still in release queues when CommandPoolManager::RecycleCommandPool() is called for destroyed CommandPoolManager object.");
     LOG_INFO_MESSAGE(m_Name, " allocated descriptor pool count: ", m_CmdPools.size());
     m_CmdPools.clear();
 }
diff --git a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
index c13982d4..40cab0ad 100644
--- a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
@@ -80,9 +80,12 @@ DeviceContextVkImpl::DeviceContextVkImpl(IReferenceCounters*                   p
     // potentially running in another thread
     m_CmdPool
     {
-        pDeviceVkImpl->GetLogicalDevice().GetSharedPtr(),
-        pDeviceVkImpl->GetCommandQueue(CommandQueueId).GetQueueFamilyIndex(),
-        VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT
+        new VulkanUtilities::VulkanCommandBufferPool
+        {
+            pDeviceVkImpl->GetLogicalDevice().GetSharedPtr(),
+            pDeviceVkImpl->GetCommandQueue(CommandQueueId).GetQueueFamilyIndex(),
+            VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT
+        }
     },
     // Upload heap must always be thread-safe as Finish() may be called from another thread
     m_UploadHeap
@@ -163,30 +166,26 @@ DeviceContextVkImpl::~DeviceContextVkImpl()
     DEV_CHECK_ERR(m_DynamicDescrSetAllocator.GetAllocatedPoolCount() == 0, "All allocated dynamic descriptor set pools must have been released at this point");
     // clang-format on
 
-    auto VkCmdPool = m_CmdPool.Release();
-    m_pDevice->SafeReleaseDeviceObject(std::move(VkCmdPool), ~Uint64{0});
-
-    // clang-format off
-    m_pDevice->SafeReleaseDeviceObject(std::move(m_GenerateMipsHelper), ~Uint64{0});
-    m_pDevice->SafeReleaseDeviceObject(std::move(m_GenerateMipsSRB),    ~Uint64{0});
-    m_pDevice->SafeReleaseDeviceObject(std::move(m_DummyVB),            ~Uint64{0});
-    // clang-format on
+    // NB: If there are any command buffers in the release queue, they will always be returned to the pool
+    //     before the pool itself is released because the pool will always end up later in the queue,
+    //     so we do not need to idle the GPU.
+    //     Also note that command buffers are disposed directly into the release queue, but
+    //     the command pool goes into the stale objects queue and is moved into the release queue
+    //     when the next command buffer is submitted.
+    m_pDevice->SafeReleaseDeviceObject(std::move(m_CmdPool), ~Uint64{0});
 
-    // The main reason we need to idle the GPU is because we need to make sure that all command buffers are returned to the
-    // pool. Upload heap, dynamic heap and dynamic descriptor manager return their resources to global managers and
-    // do not really need to wait for GPU to idle.
-    m_pDevice->IdleGPU();
-    DEV_CHECK_ERR(m_CmdPool.DvpGetBufferCounter() == 0, "All command buffers must have been returned to the pool");
+    // NB: Upload heap, dynamic heap and dynamic descriptor manager return their resources to
+    //     global managers and do not need to wait for GPU to idle.
 }
 
 void DeviceContextVkImpl::DisposeVkCmdBuffer(Uint32 CmdQueue, VkCommandBuffer vkCmdBuff, Uint64 FenceValue)
 {
     VERIFY_EXPR(vkCmdBuff != VK_NULL_HANDLE);
-    class CmdBufferDeleter
+    class CmdBufferRecycler
     {
     public:
         // clang-format off
-        CmdBufferDeleter(VkCommandBuffer                           _vkCmdBuff, 
+        CmdBufferRecycler(VkCommandBuffer                           _vkCmdBuff, 
                          VulkanUtilities::VulkanCommandBufferPool& _Pool) noexcept :
             vkCmdBuff {_vkCmdBuff},
             Pool      {&_Pool    }
@@ -194,11 +193,11 @@ void DeviceContextVkImpl::DisposeVkCmdBuffer(Uint32 CmdQueue, VkCommandBuffer vk
             VERIFY_EXPR(vkCmdBuff != VK_NULL_HANDLE);
         }
 
-        CmdBufferDeleter             (const CmdBufferDeleter&)  = delete;
-        CmdBufferDeleter& operator = (const CmdBufferDeleter&)  = delete;
-        CmdBufferDeleter& operator = (      CmdBufferDeleter&&) = delete;
+        CmdBufferRecycler             (const CmdBufferRecycler&)  = delete;
+        CmdBufferRecycler& operator = (const CmdBufferRecycler&)  = delete;
+        CmdBufferRecycler& operator = (      CmdBufferRecycler&&) = delete;
 
-        CmdBufferDeleter(CmdBufferDeleter&& rhs) noexcept : 
+        CmdBufferRecycler(CmdBufferRecycler&& rhs) noexcept : 
             vkCmdBuff {rhs.vkCmdBuff},
             Pool      {rhs.Pool     }
         {
@@ -207,21 +206,23 @@ void DeviceContextVkImpl::DisposeVkCmdBuffer(Uint32 CmdQueue, VkCommandBuffer vk
         }
         // clang-format on
 
-        ~CmdBufferDeleter()
+        ~CmdBufferRecycler()
         {
             if (Pool != nullptr)
             {
-                Pool->FreeCommandBuffer(std::move(vkCmdBuff));
+                Pool->RecycleCommandBuffer(std::move(vkCmdBuff));
             }
         }
 
     private:
-        VkCommandBuffer                           vkCmdBuff;
-        VulkanUtilities::VulkanCommandBufferPool* Pool;
+        VkCommandBuffer                           vkCmdBuff = VK_NULL_HANDLE;
+        VulkanUtilities::VulkanCommandBufferPool* Pool      = nullptr;
     };
 
+    // Discard command buffer directly to the release queue since we know exactly which queue it was submitted to
+    // as well as the associated FenceValue.
     auto& ReleaseQueue = m_pDevice->GetReleaseQueue(CmdQueue);
-    ReleaseQueue.DiscardResource(CmdBufferDeleter{vkCmdBuff, m_CmdPool}, FenceValue);
+    ReleaseQueue.DiscardResource(CmdBufferRecycler{vkCmdBuff, *m_CmdPool}, FenceValue);
 }
 
 inline void DeviceContextVkImpl::DisposeCurrentCmdBuffer(Uint32 CmdQueue, Uint64 FenceValue)
diff --git a/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp
index e1ce9efa..8cdfceb6 100644
--- a/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp
@@ -122,7 +122,7 @@ RenderDeviceVkImpl::RenderDeviceVkImpl(IReferenceCounters*
     },
     m_TransientCmdPoolMgr
     {
-        *this,
+        GetLogicalDevice(),
         "Transient command buffer pool manager",
         CmdQueues[0]->GetQueueFamilyIndex(),
         VK_COMMAND_POOL_CREATE_TRANSIENT_BIT
@@ -327,7 +327,9 @@ void RenderDeviceVkImpl::AllocateTransientCmdPool(VulkanUtilities::CommandPoolWr
 }
 
 
-void RenderDeviceVkImpl::ExecuteAndDisposeTransientCmdBuff(Uint32 QueueIndex, VkCommandBuffer vkCmdBuff, VulkanUtilities::CommandPoolWrapper&& CmdPool)
+void RenderDeviceVkImpl::ExecuteAndDisposeTransientCmdBuff(Uint32                                QueueIndex,
+                                                           VkCommandBuffer                       vkCmdBuff,
+                                                           VulkanUtilities::CommandPoolWrapper&& CmdPool)
 {
     VERIFY_EXPR(vkCmdBuff != VK_NULL_HANDLE);
 
@@ -373,7 +375,71 @@ void RenderDeviceVkImpl::ExecuteAndDisposeTransientCmdBuff(Uint32 QueueIndex, Vk
                            FenceValue = pCmdQueueVk->Submit(SubmitInfo);
                        } //
     );
-    m_TransientCmdPoolMgr.SafeReleaseCommandPool(std::move(CmdPool), QueueIndex, FenceValue);
+
+    class TransientCmdPoolRecycler
+    {
+    public:
+        TransientCmdPoolRecycler(const VulkanUtilities::VulkanLogicalDevice& _LogicalDevice,
+                                 CommandPoolManager&                         _CmdPoolMgr,
+                                 VulkanUtilities::CommandPoolWrapper&&       _Pool,
+                                 VkCommandBuffer&&                           _vkCmdBuffer) :
+            // clang-format off
+            LogicalDevice{_LogicalDevice         },
+            CmdPoolMgr   {&_CmdPoolMgr           },
+            Pool         {std::move(_Pool)       },
+            vkCmdBuffer  {std::move(_vkCmdBuffer)}
+        // clang-format on
+        {
+            VERIFY_EXPR(Pool != VK_NULL_HANDLE && vkCmdBuffer != VK_NULL_HANDLE);
+            _vkCmdBuffer = VK_NULL_HANDLE;
+        }
+
+        // clang-format off
+        TransientCmdPoolRecycler             (const TransientCmdPoolRecycler&)  = delete;
+        TransientCmdPoolRecycler& operator = (const TransientCmdPoolRecycler&)  = delete;
+        TransientCmdPoolRecycler& operator = (      TransientCmdPoolRecycler&&) = delete;
+
+        TransientCmdPoolRecycler(TransientCmdPoolRecycler&& rhs) :
+            LogicalDevice{rhs.LogicalDevice         },
+            CmdPoolMgr   {rhs.CmdPoolMgr            },
+            Pool         {std::move(rhs.Pool)       },
+            vkCmdBuffer  {std::move(rhs.vkCmdBuffer)}
+        {
+            rhs.CmdPoolMgr  = nullptr;
+            rhs.vkCmdBuffer = VK_NULL_HANDLE;
+        }
+        // clang-format on
+
+        ~TransientCmdPoolRecycler()
+        {
+            if (CmdPoolMgr != nullptr)
+            {
+                LogicalDevice.FreeCommandBuffer(Pool, vkCmdBuffer);
+                CmdPoolMgr->RecycleCommandPool(std::move(Pool));
+            }
+        }
+
+    private:
+        const VulkanUtilities::VulkanLogicalDevice& LogicalDevice;
+
+        CommandPoolManager*                 CmdPoolMgr = nullptr;
+        VulkanUtilities::CommandPoolWrapper Pool;
+        VkCommandBuffer                     vkCmdBuffer = VK_NULL_HANDLE;
+    };
+
+    // Discard command pool directly to the release queue since we know exactly which queue it was submitted to
+    // as well as the associated FenceValue
+    // clang-format off
+    GetReleaseQueue(QueueIndex).DiscardResource(
+        TransientCmdPoolRecycler
+        {
+            GetLogicalDevice(),
+            m_TransientCmdPoolMgr,
+            std::move(CmdPool),
+            std::move(vkCmdBuff)
+        },
+        FenceValue);
+    // clang-format on
 }
 
 void RenderDeviceVkImpl::SubmitCommandBuffer(Uint32                                                 QueueIndex,
diff --git a/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanCommandBufferPool.cpp b/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanCommandBufferPool.cpp
index fb2c12ad..e4b5b3d8 100644
--- a/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanCommandBufferPool.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanCommandBufferPool.cpp
@@ -49,15 +49,18 @@ VulkanCommandBufferPool::VulkanCommandBufferPool(std::shared_ptr<const VulkanLog
 
     m_CmdPool = m_LogicalDevice->CreateCommandPool(CmdPoolCI);
     DEV_CHECK_ERR(m_CmdPool != VK_NULL_HANDLE, "Failed to create vulkan command pool");
-#ifdef DILIGENT_DEVELOPMENT
-    m_BuffCounter = 0;
-#endif
 }
 
 VulkanCommandBufferPool::~VulkanCommandBufferPool()
 {
+    DEV_CHECK_ERR(m_BuffCounter == 0, m_BuffCounter,
+                  " command buffer(s) have not been returned to the pool. If there are outstanding references to these "
+                  "buffers in release queues, VulkanCommandBufferPool::RecycleCommandBuffer() will crash when attempting to "
+                  "return the buffer to the pool.");
+
+    for (auto CmdBuff : m_CmdBuffers)
+        m_LogicalDevice->FreeCommandBuffer(m_CmdPool, CmdBuff);
     m_CmdPool.Release();
-    DEV_CHECK_ERR(m_BuffCounter == 0, m_BuffCounter, " command buffer(s) have not been returned to the pool. If there are outstanding references to these buffers in release queues, FreeCommandBuffer() will crash when attempting to return a buffer to the pool.");
 }
 
 VkCommandBuffer VulkanCommandBufferPool::GetCommandBuffer(const char* DebugName)
@@ -114,7 +117,7 @@ VkCommandBuffer VulkanCommandBufferPool::GetCommandBuffer(const char* DebugName)
     return CmdBuffer;
 }
 
-void VulkanCommandBufferPool::FreeCommandBuffer(VkCommandBuffer&& CmdBuffer)
+void VulkanCommandBufferPool::RecycleCommandBuffer(VkCommandBuffer&& CmdBuffer)
 {
     std::lock_guard<std::mutex> Lock{m_Mutex};
     m_CmdBuffers.emplace_back(CmdBuffer);
@@ -124,11 +127,4 @@ void VulkanCommandBufferPool::FreeCommandBuffer(VkCommandBuffer&& CmdBuffer)
 #endif
 }
 
-CommandPoolWrapper&& VulkanCommandBufferPool::Release()
-{
-    m_LogicalDevice.reset();
-    m_CmdBuffers.clear();
-    return std::move(m_CmdPool);
-}
-
 } // namespace VulkanUtilities
diff --git a/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanLogicalDevice.cpp b/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanLogicalDevice.cpp
index ebddece7..5b4b6f5d 100644
--- a/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanLogicalDevice.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanLogicalDevice.cpp
@@ -463,6 +463,11 @@ void VulkanLogicalDevice::FreeDescriptorSet(VkDescriptorPool Pool, VkDescriptorS
 }
 
 
+void VulkanLogicalDevice::FreeCommandBuffer(VkCommandPool Pool, VkCommandBuffer CmdBuffer) const
+{
+    VERIFY_EXPR(Pool != VK_NULL_HANDLE && CmdBuffer != VK_NULL_HANDLE);
+    vkFreeCommandBuffers(m_VkDevice, Pool, 1, &CmdBuffer);
+}
 
 
 VkMemoryRequirements VulkanLogicalDevice::GetBufferMemoryRequirements(VkBuffer vkBuffer) const