Refactored device object release queues

28 files changed, 198 insertions, 244 deletions

diff --git a/Graphics/GraphicsEngineVulkan/CMakeLists.txt b/Graphics/GraphicsEngineVulkan/CMakeLists.txt
index e920fcae..0a67939f 100644
--- a/Graphics/GraphicsEngineVulkan/CMakeLists.txt
+++ b/Graphics/GraphicsEngineVulkan/CMakeLists.txt
@@ -176,7 +176,7 @@ set(PRIVATE_DEPENDENCIES
     BuildSettings 
     Common 
     TargetPlatform
-    GraphicsEngine
+    GraphicsEngineNextGenBase
     GLSLTools
 )
 
diff --git a/Graphics/GraphicsEngineVulkan/include/CommandQueueVkImpl.h b/Graphics/GraphicsEngineVulkan/include/CommandQueueVkImpl.h
index 5e31bf01..51eff69f 100644
--- a/Graphics/GraphicsEngineVulkan/include/CommandQueueVkImpl.h
+++ b/Graphics/GraphicsEngineVulkan/include/CommandQueueVkImpl.h
@@ -51,18 +51,20 @@ public:
     virtual void QueryInterface( const Diligent::INTERFACE_ID& IID, IObject** ppInterface )override;
 
 	// Returns the fence value that will be signaled next time
-    virtual UINT64 GetNextFenceValue()override final { return m_NextFenceValue; }
+    virtual Uint64 GetNextFenceValue()override final { return m_NextFenceValue; }
 
-	// Executes a given command buffer
-	virtual Uint64 ExecuteCommandBuffer(VkCommandBuffer cmdBuffer)override final;
+	// Submits a given command buffer to the queue
+	virtual Uint64 Submit(VkCommandBuffer cmdBuffer)override final;
 
-    virtual Uint64 ExecuteCommandBuffer(const VkSubmitInfo& SubmitInfo)override final;
+    virtual Uint64 Submit(const VkSubmitInfo& SubmitInfo)override final;
+
+    virtual void Present(const VkPresentInfoKHR& PresentInfo)override final;
 
     virtual VkQueue GetVkQueue()override final{return m_VkQueue;}
 
     virtual uint32_t GetQueueFamilyIndex()override final { return m_QueueFamilyIndex; }
     
-    virtual Uint64 IdleGPU()override final;
+    virtual Uint64 WaitForIdle()override final;
 
     virtual Uint64 GetCompletedFenceValue()override final;
 
diff --git a/Graphics/GraphicsEngineVulkan/include/DescriptorPoolManager.h b/Graphics/GraphicsEngineVulkan/include/DescriptorPoolManager.h
index 2777a78e..4df8d9f7 100644
--- a/Graphics/GraphicsEngineVulkan/include/DescriptorPoolManager.h
+++ b/Graphics/GraphicsEngineVulkan/include/DescriptorPoolManager.h
@@ -40,9 +40,11 @@ class DescriptorPoolAllocation
 {
 public:
     DescriptorPoolAllocation(VkDescriptorSet                        _Set,
+                             Uint64                                 _CmdQueueMask,
                              VulkanUtilities::VulkanDescriptorPool& _ParentPool,
                              DescriptorPoolManager&                 _ParentPoolMgr)noexcept :
         Set          (_Set),
+        CmdQueueMask(_CmdQueueMask),
         ParentPool   (&_ParentPool),
         ParentPoolMgr(&_ParentPoolMgr)
     {}
@@ -53,10 +55,12 @@ public:
 
     DescriptorPoolAllocation(DescriptorPoolAllocation&& rhs)noexcept : 
         Set          (rhs.Set),
+        CmdQueueMask (rhs.CmdQueueMask),
         ParentPool   (rhs.ParentPool),
         ParentPoolMgr(rhs.ParentPoolMgr)
     {
         rhs.Set           = VK_NULL_HANDLE;
+        rhs.CmdQueueMask  = 0;
         rhs.ParentPool    = nullptr;
         rhs.ParentPoolMgr = nullptr;
     }
@@ -66,10 +70,12 @@ public:
         Release();
 
         Set           = rhs.Set;
+        CmdQueueMask  = rhs.CmdQueueMask;
         ParentPool    = rhs.ParentPool;
         ParentPoolMgr = rhs.ParentPoolMgr;
 
         rhs.Set           = VK_NULL_HANDLE;
+        rhs.CmdQueueMask  = 0;
         rhs.ParentPool    = nullptr;
         rhs.ParentPoolMgr = nullptr;
         
@@ -92,6 +98,7 @@ public:
 
 private:
     VkDescriptorSet                        Set           = VK_NULL_HANDLE;
+    Uint64                                 CmdQueueMask  = 0;
     VulkanUtilities::VulkanDescriptorPool* ParentPool    = nullptr;
     DescriptorPoolManager*                 ParentPoolMgr = nullptr;
 };
@@ -125,7 +132,7 @@ public:
     DescriptorPoolManager& operator = (const DescriptorPoolManager&) = delete;
     DescriptorPoolManager& operator = (DescriptorPoolManager&&)      = delete;
 
-    DescriptorPoolAllocation Allocate(VkDescriptorSetLayout SetLayout);
+    DescriptorPoolAllocation Allocate(Uint64 CommandQueueMask, VkDescriptorSetLayout SetLayout);
     void DisposeAllocations(uint64_t FenceValue);
     void ReleaseStaleAllocations(uint64_t LastCompletedFence);
     
diff --git a/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h b/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h
index 06a6a744..6249c1d6 100644
--- a/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h
+++ b/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h
@@ -56,6 +56,7 @@ public:
                         bool                                  bIsDeferred,
                         const EngineVkAttribs&                Attribs,
                         Uint32                                ContextId,
+                        Uint32                                CommandQueueId,
                         std::shared_ptr<GenerateMipsVkHelper> GenerateMipsHelper);
     ~DeviceContextVkImpl();
     
@@ -169,7 +170,7 @@ public:
     {
         // Descriptor pools are externally synchronized, meaning that the application must not allocate 
         // and/or free descriptor sets from the same pool in multiple threads simultaneously (13.2.3)
-        return m_DynamicDescriptorPool.Allocate(SetLayout);
+        return m_DynamicDescriptorPool.Allocate( Uint64{1} << Uint64{m_CommandQueueId}, SetLayout);
     }
 
     VulkanDynamicAllocation AllocateDynamicSpace(Uint32 SizeInBytes, Uint32 Alignment);
@@ -240,6 +241,7 @@ private:
     FixedBlockMemoryAllocator m_CmdListAllocator;
 
     const Uint32 m_ContextId;
+    const Uint32 m_CommandQueueId;
 
     VulkanUtilities::VulkanCommandBufferPool m_CmdPool;
 
@@ -282,7 +284,7 @@ private:
 
 
 
-    VulkanUploadHeap m_UploadHeap;
+    VulkanUploadHeap      m_UploadHeap;
     DescriptorPoolManager m_DynamicDescriptorPool;
 
     // Number of the command buffer currently being recorded by the context and that will
diff --git a/Graphics/GraphicsEngineVulkan/include/FramebufferCache.h b/Graphics/GraphicsEngineVulkan/include/FramebufferCache.h
index 97871607..cfac38b4 100644
--- a/Graphics/GraphicsEngineVulkan/include/FramebufferCache.h
+++ b/Graphics/GraphicsEngineVulkan/include/FramebufferCache.h
@@ -56,6 +56,7 @@ public:
         Uint32       NumRenderTargets;
         VkImageView  DSV;
         VkImageView  RTVs[MaxRenderTargets];
+        Uint64       CommandQueueMask;
 
         bool operator == (const FramebufferCacheKey &rhs)const;
         size_t GetHash()const;
diff --git a/Graphics/GraphicsEngineVulkan/include/PipelineLayout.h b/Graphics/GraphicsEngineVulkan/include/PipelineLayout.h
index 9d2ef248..a2ebd353 100644
--- a/Graphics/GraphicsEngineVulkan/include/PipelineLayout.h
+++ b/Graphics/GraphicsEngineVulkan/include/PipelineLayout.h
@@ -47,7 +47,7 @@ public:
     static VkDescriptorType GetVkDescriptorType(const SPIRVShaderResourceAttribs &Res);
 
     PipelineLayout();
-    void Release(RenderDeviceVkImpl* pDeviceVkImpl);
+    void Release(RenderDeviceVkImpl* pDeviceVkImpl, Uint64 CommandQueueMask);
     void Finalize(const VulkanUtilities::VulkanLogicalDevice& LogicalDevice);
 
     VkPipelineLayout GetVkPipelineLayout()const{return m_LayoutMgr.GetVkPipelineLayout();}
@@ -153,7 +153,7 @@ private:
             ~DescriptorSetLayout();
             void AddBinding(const VkDescriptorSetLayoutBinding& Binding, IMemoryAllocator& MemAllocator);
             void Finalize(const VulkanUtilities::VulkanLogicalDevice& LogicalDevice, IMemoryAllocator& MemAllocator, VkDescriptorSetLayoutBinding* pNewBindings);
-            void Release(RenderDeviceVkImpl* pRenderDeviceVk, IMemoryAllocator& MemAllocator);
+            void Release(RenderDeviceVkImpl* pRenderDeviceVk, IMemoryAllocator& MemAllocator, Uint64 CommandQueueMask);
 
             bool operator == (const DescriptorSetLayout& rhs)const;
             bool operator != (const DescriptorSetLayout& rhs)const{return !(*this == rhs);}
@@ -173,7 +173,7 @@ private:
         DescriptorSetLayoutManager& operator= (DescriptorSetLayoutManager&&)      = delete;
         
         void Finalize(const VulkanUtilities::VulkanLogicalDevice &LogicalDevice);
-        void Release(RenderDeviceVkImpl* pRenderDeviceVk);
+        void Release(RenderDeviceVkImpl* pRenderDeviceVk, Uint64 CommandQueueMask);
 
               DescriptorSetLayout& GetDescriptorSet(SHADER_VARIABLE_TYPE VarType)      { return m_DescriptorSetLayouts[VarType == SHADER_VARIABLE_TYPE_DYNAMIC ? 1 : 0]; }
         const DescriptorSetLayout& GetDescriptorSet(SHADER_VARIABLE_TYPE VarType)const { return m_DescriptorSetLayouts[VarType == SHADER_VARIABLE_TYPE_DYNAMIC ? 1 : 0]; }
diff --git a/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h b/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h
index 49db82b9..aff6fbae 100644
--- a/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h
+++ b/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h
@@ -29,6 +29,7 @@
 
 #include "RenderDeviceVk.h"
 #include "RenderDeviceBase.h"
+#include "RenderDeviceNextGenBase.h"
 #include "DescriptorPoolManager.h"
 #include "VulkanDynamicHeap.h"
 #include "Atomics.h"
@@ -51,10 +52,10 @@ namespace Diligent
 {
 
 /// Implementation of the Diligent::IRenderDeviceVk interface
-class RenderDeviceVkImpl final : public RenderDeviceBase<IRenderDeviceVk>
+class RenderDeviceVkImpl final : public RenderDeviceNextGenBase<RenderDeviceBase<IRenderDeviceVk>, ICommandQueueVk>
 {
 public:
-    using TRenderDeviceBase = RenderDeviceBase<IRenderDeviceVk>;
+    using TRenderDeviceBase = RenderDeviceNextGenBase<RenderDeviceBase<IRenderDeviceVk>, ICommandQueueVk>;
 
     RenderDeviceVkImpl( IReferenceCounters*     pRefCounters, 
                         IMemoryAllocator&       RawMemAllocator, 
@@ -88,40 +89,36 @@ public:
 
     virtual void CreateBufferFromVulkanResource(VkBuffer vkBuffer, const BufferDesc& BuffDesc, IBuffer** ppBuffer)override final;
 
-    Uint64 GetCompletedFenceValue();
-	virtual Uint64 GetNextFenceValue() override final
+    virtual Uint64 GetCompletedFenceValue(Uint32 QueueIndex) override final
     {
-        return m_pCommandQueue->GetNextFenceValue();
+        return m_CommandQueues[QueueIndex].CmdQueue->GetCompletedFenceValue();
     }
 
-	Uint64 GetCurrentFrameNumber()const {return static_cast<Uint64>(m_FrameNumber);}
-    virtual Bool IsFenceSignaled(Uint64 FenceValue) override final;
+    virtual Uint64 GetNextFenceValue(Uint32 QueueIndex) override final
+    {
+        return m_CommandQueues[QueueIndex].CmdQueue->GetNextFenceValue();
+    }
 
-    ICommandQueueVk *GetCmdQueue(){return m_pCommandQueue;}
-    
-    // Idles GPU and returns fence value that was signaled
-	Uint64 IdleGPU(bool ReleaseStaleObjects);
+    virtual Bool IsFenceSignaled(Uint32 QueueIndex, Uint64 FenceValue) override final
+    {
+        return FenceValue <= GetCompletedFenceValue(QueueIndex);
+    }
+
+    // Idles GPU
+	void IdleGPU(bool ReleaseStaleObjects);
     // pImmediateCtx parameter is only used to make sure the command buffer is submitted from the immediate context
     // The method returns fence value associated with the submitted command buffer
     Uint64 ExecuteCommandBuffer(const VkSubmitInfo &SubmitInfo, class DeviceContextVkImpl* pImmediateCtx, std::vector<std::pair<Uint64, RefCntAutoPtr<IFence> > >* pSignalFences);
 
     void AllocateTransientCmdPool(VulkanUtilities::CommandPoolWrapper& CmdPool, VkCommandBuffer& vkCmdBuff, const Char* DebugPoolName = nullptr);
-    void ExecuteAndDisposeTransientCmdBuff(VkCommandBuffer vkCmdBuff, VulkanUtilities::CommandPoolWrapper&& CmdPool);
+    void ExecuteAndDisposeTransientCmdBuff(Uint32 QueueIndex, VkCommandBuffer vkCmdBuff, VulkanUtilities::CommandPoolWrapper&& CmdPool);
 
-    template<typename ObjectType>
-    void SafeReleaseVkObject(ObjectType&& Object)
-    {
-        m_ReleaseQueue.SafeReleaseResource(std::move(Object), m_NextCmdBuffNumber);
-    }
-
-    ResourceReleaseQueue<DynamicStaleResourceWrapper>& GetReleaseQueue(){return m_ReleaseQueue;}
-    
     void FinishFrame(bool ReleaseAllResources);
     virtual void FinishFrame()override final { FinishFrame(false); }
 
-    DescriptorPoolAllocation AllocateDescriptorSet(VkDescriptorSetLayout SetLayout)
+    DescriptorPoolAllocation AllocateDescriptorSet(Uint64 CommandQueueMask, VkDescriptorSetLayout SetLayout)
     {
-        return m_MainDescriptorPool.Allocate(SetLayout);
+        return m_MainDescriptorPool.Allocate(CommandQueueMask, SetLayout);
     }
 
     std::shared_ptr<const VulkanUtilities::VulkanInstance> GetVulkanInstance()const{return m_VulkanInstance;}
@@ -140,7 +137,6 @@ public:
      
 private:
     virtual void TestTextureFormat( TEXTURE_FORMAT TexFormat )override final;
-    void ProcessStaleResources(Uint64 SubmittedCmdBufferNumber, Uint64 SubmittedFenceValue, Uint64 CompletedFenceValue);
 
     // Submits command buffer for execution to the command queue
     // Returns the submitted command buffer number and the fence value
@@ -152,54 +148,19 @@ private:
     std::shared_ptr<VulkanUtilities::VulkanInstance>        m_VulkanInstance;
     std::unique_ptr<VulkanUtilities::VulkanPhysicalDevice>  m_PhysicalDevice;
     std::shared_ptr<VulkanUtilities::VulkanLogicalDevice>   m_LogicalVkDevice;
-    
-    std::mutex                      m_SubmitCmdBufferMutex;
-    RefCntAutoPtr<ICommandQueueVk>  m_pCommandQueue;
 
     EngineVkAttribs m_EngineAttribs;
 
-	Atomics::AtomicInt64 m_FrameNumber;
-    Atomics::AtomicInt64 m_NextCmdBuffNumber;
-    
-    // The following basic requirement guarantees correctness of resource deallocation:
-    //
-    //        A resource is never released before the last draw command referencing it is invoked on the immediate context
-    //
-
-    //
-    // CPU
-    //                       Last Reference
-    //                        of resource X
-    //                             |
-    //                             |     Submit Cmd       Submit Cmd            Submit Cmd
-    //                             |      List N           List N+1              List N+2
-    //                             V         |                |                     |
-    //    NextFenceValue       |   *  N      |      N+1       |          N+2        |
-    //
-    //
-    //    CompletedFenceValue       |     N-3      |      N-2      |        N-1        |        N       |
-    //                              .              .               .                   .                .
-    // -----------------------------.--------------.---------------.-------------------.----------------.-------------
-    //                              .              .               .                   .                .
-    //       
-    // GPU                          | Cmd List N-2 | Cmd List N-1  |    Cmd List N     |   Cmd List N+1 |
-    //                                                                                 |
-    //                                                                                 |
-    //                                                                          Resource X can
-    //                                                                           be released
-
-    FramebufferCache m_FramebufferCache;
-    RenderPassCache  m_RenderPassCache;
-
+    FramebufferCache      m_FramebufferCache;
+    RenderPassCache       m_RenderPassCache;
     DescriptorPoolManager m_MainDescriptorPool;
 
     // These one-time command pools are used by buffer and texture constructors to
     // issue copy commands. Vulkan requires that every command pool is used by one thread 
     // at a time, so every constructor must allocate command buffer from its own pool.
-    CommandPoolManager m_TransientCmdPoolMgr;
+    CommandPoolManager    m_TransientCmdPoolMgr;
 
     VulkanUtilities::VulkanMemoryManager m_MemoryMgr;
-    ResourceReleaseQueue<DynamicStaleResourceWrapper> m_ReleaseQueue;
 
     VulkanDynamicMemoryManager m_DynamicMemoryManager;
 };
diff --git a/Graphics/GraphicsEngineVulkan/include/SamplerVkImpl.h b/Graphics/GraphicsEngineVulkan/include/SamplerVkImpl.h
index d8adb0a9..418fd8da 100644
--- a/Graphics/GraphicsEngineVulkan/include/SamplerVkImpl.h
+++ b/Graphics/GraphicsEngineVulkan/include/SamplerVkImpl.h
@@ -53,6 +53,7 @@ private:
     friend class ShaderVkImpl;
     /// Vk sampler handle
     VulkanUtilities::SamplerWrapper m_VkSampler;
+    static constexpr Uint64 m_CommandQueueMask = ~Uint64{0};
 };
 
 }
diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanDynamicHeap.h b/Graphics/GraphicsEngineVulkan/include/VulkanDynamicHeap.h
index 79a95abe..31f291de 100644
--- a/Graphics/GraphicsEngineVulkan/include/VulkanDynamicHeap.h
+++ b/Graphics/GraphicsEngineVulkan/include/VulkanDynamicHeap.h
@@ -108,7 +108,8 @@ public:
 
     VulkanDynamicMemoryManager(IMemoryAllocator&         Allocator, 
                                class RenderDeviceVkImpl& DeviceVk, 
-                               Uint32                    Size);
+                               Uint32                    Size,
+                               Uint64                    CommandQueueMask);
     ~VulkanDynamicMemoryManager();
 
     VulkanDynamicMemoryManager            (const VulkanDynamicMemoryManager&)  = delete;
@@ -131,7 +132,7 @@ private:
     VulkanUtilities::DeviceMemoryWrapper m_BufferMemory;
     Uint8*                               m_CPUAddress;
     const VkDeviceSize                   m_DefaultAlignment;
-
+    const Uint64                         m_CommandQueueMask;
     OffsetType  m_TotalPeakSize    = 0;
 };
 
diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanUploadHeap.h b/Graphics/GraphicsEngineVulkan/include/VulkanUploadHeap.h
index 3e29b87b..c1d0e5a9 100644
--- a/Graphics/GraphicsEngineVulkan/include/VulkanUploadHeap.h
+++ b/Graphics/GraphicsEngineVulkan/include/VulkanUploadHeap.h
@@ -66,7 +66,7 @@ public:
     ~VulkanUploadHeap();
 
     VulkanUploadAllocation Allocate(size_t SizeInBytes, size_t Alignment);
-    void DiscardAllocations(uint64_t FenceValue);
+    void DiscardAllocations(Uint32 CommandQueueIndex, uint64_t FenceValue);
 
     size_t GetStaleAllocationsCount()const
     {
diff --git a/Graphics/GraphicsEngineVulkan/interface/CommandQueueVk.h b/Graphics/GraphicsEngineVulkan/interface/CommandQueueVk.h
index 3a5fb616..df4cfb60 100644
--- a/Graphics/GraphicsEngineVulkan/interface/CommandQueueVk.h
+++ b/Graphics/GraphicsEngineVulkan/interface/CommandQueueVk.h
@@ -41,17 +41,20 @@ class ICommandQueueVk : public Diligent::IObject
 {
 public:
 	/// Returns the fence value that will be signaled next time
-	virtual UINT64 GetNextFenceValue() = 0;
+	virtual Uint64 GetNextFenceValue() = 0;
 
-	/// Executes a given command buffer
+	/// Submits a given command buffer to the command queue
 
-    /// \return Fence value associated with the executed command buffer
-	virtual Uint64 ExecuteCommandBuffer(VkCommandBuffer cmdBuffer) = 0;
+    /// \return Fence value associated with the submitted command buffer
+	virtual Uint64 Submit(VkCommandBuffer cmdBuffer) = 0;
 
     /// Submits a given chunk of work to the command queue
 
-    /// \return Fence value associated with the executed command buffer
-    virtual Uint64 ExecuteCommandBuffer(const VkSubmitInfo& SubmitInfo) = 0;
+    /// \return Fence value associated with the submitted command buffer
+    virtual Uint64 Submit(const VkSubmitInfo& SubmitInfo) = 0;
+
+    /// Presents the current swap chain image on the screen
+    virtual void Present(const VkPresentInfoKHR& PresentInfo) = 0;
 
     /// Returns Vulkan command queue. May return VK_NULL_HANDLE if queue is anavailable
     virtual VkQueue GetVkQueue() = 0;
@@ -65,7 +68,7 @@ public:
     /// Blocks execution until all pending GPU commands are complete
 
     /// \return Last completed fence value
-    virtual UINT64 IdleGPU() = 0;
+    virtual Uint64 WaitForIdle() = 0;
 
     /// Signals the given fence
     virtual void SignalFence(VkFence vkFence) = 0;
diff --git a/Graphics/GraphicsEngineVulkan/interface/RenderDeviceVk.h b/Graphics/GraphicsEngineVulkan/interface/RenderDeviceVk.h
index 82199520..1bb66d27 100644
--- a/Graphics/GraphicsEngineVulkan/interface/RenderDeviceVk.h
+++ b/Graphics/GraphicsEngineVulkan/interface/RenderDeviceVk.h
@@ -44,11 +44,14 @@ public:
     virtual VkDevice GetVkDevice() = 0;
 
     /// Returns the fence value that will be signaled by the GPU command queue next
-    virtual Uint64 GetNextFenceValue() = 0;
+    virtual Uint64 GetNextFenceValue(Uint32 QueueIndex) = 0;
+
+    /// Returns the last completed fence value for the given command queue
+    virtual Uint64 GetCompletedFenceValue(Uint32 QueueIndex) = 0;
 
     /// Checks if the fence value has been signaled by the GPU. True means
     /// that all associated work has been finished
-    virtual Bool IsFenceSignaled(Uint64 FenceValue) = 0;
+    virtual Bool IsFenceSignaled(Uint32 QueueIndex, Uint64 FenceValue) = 0;
 
     /// Should be called at the end of the frame when attached to existing D3D12 device
     /// Otherwise the method is automatically called before present
diff --git a/Graphics/GraphicsEngineVulkan/src/BufferViewVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/BufferViewVkImpl.cpp
index 355aae7a..31c8be7a 100644
--- a/Graphics/GraphicsEngineVulkan/src/BufferViewVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/BufferViewVkImpl.cpp
@@ -42,7 +42,7 @@ BufferViewVkImpl::BufferViewVkImpl( IReferenceCounters*                  pRefCou
 
 BufferViewVkImpl::~BufferViewVkImpl()
 {
-    m_pDevice->SafeReleaseVkObject(std::move(m_BuffView));
+    m_pDevice->SafeReleaseDeviceObject(std::move(m_BuffView), GetBufferVk()->GetDesc().CommandQueueMask);
 }
 
 IMPLEMENT_QUERY_INTERFACE( BufferViewVkImpl, IID_BufferViewVk, TBufferViewBase )
diff --git a/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp
index ba9c2ca6..e5563c5d 100644
--- a/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp
@@ -221,7 +221,8 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters*        pRefCounters,
             BuffCopy.size = VkBuffCI.size;
             vkCmdCopyBuffer(vkCmdBuff, StagingBuffer, m_VulkanBuffer, 1, &BuffCopy);
 
-	        pRenderDeviceVk->ExecuteAndDisposeTransientCmdBuff(vkCmdBuff, std::move(CmdPool));
+            Uint32 QueueIndex = 0;
+	        pRenderDeviceVk->ExecuteAndDisposeTransientCmdBuff(QueueIndex, vkCmdBuff, std::move(CmdPool));
 
 
             // After command buffer is submitted, safe-release staging resources. This strategy
@@ -248,8 +249,8 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters*        pRefCounters,
             //              |            |                                                |   - {F+1, StagingBuffer} -> Release Queue 
             //              |            |                                                | 
 
-            pRenderDeviceVk->SafeReleaseVkObject(std::move(StagingBuffer));
-            pRenderDeviceVk->SafeReleaseVkObject(std::move(StagingMemoryAllocation));
+            pRenderDeviceVk->SafeReleaseDeviceObject(std::move(StagingBuffer),           Uint64{1} << Uint64{QueueIndex});
+            pRenderDeviceVk->SafeReleaseDeviceObject(std::move(StagingMemoryAllocation), Uint64{1} << Uint64{QueueIndex});
         }
         else
         {
@@ -278,9 +279,9 @@ BufferVkImpl :: ~BufferVkImpl()
 {
     // Vk object can only be destroyed when it is no longer used by the GPU
     if(m_VulkanBuffer != VK_NULL_HANDLE)
-        m_pDevice->SafeReleaseVkObject(std::move(m_VulkanBuffer));
+        m_pDevice->SafeReleaseDeviceObject(std::move(m_VulkanBuffer), m_Desc.CommandQueueMask);
     if(m_MemoryAllocation.Page != nullptr)
-        m_pDevice->SafeReleaseVkObject(std::move(m_MemoryAllocation));
+        m_pDevice->SafeReleaseDeviceObject(std::move(m_MemoryAllocation), m_Desc.CommandQueueMask);
 }
 
 IMPLEMENT_QUERY_INTERFACE( BufferVkImpl, IID_BufferVk, TBufferBase )
diff --git a/Graphics/GraphicsEngineVulkan/src/CommandQueueVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/CommandQueueVkImpl.cpp
index 6e789ee5..8ec55707 100644
--- a/Graphics/GraphicsEngineVulkan/src/CommandQueueVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/CommandQueueVkImpl.cpp
@@ -41,18 +41,22 @@ CommandQueueVkImpl::CommandQueueVkImpl(IReferenceCounters*
 
 CommandQueueVkImpl::~CommandQueueVkImpl()
 {
-    // Queues are created along with a logical device during vkCreateDevice.
-    // All queues associated with a logical device are destroyed when vkDestroyDevice 
+    // Queues are created along with the logical device during vkCreateDevice.
+    // All queues associated with the logical device are destroyed when vkDestroyDevice 
     // is called on that device.
-
 }
 
 IMPLEMENT_QUERY_INTERFACE( CommandQueueVkImpl, IID_CommandQueueVk, TBase )
 
 
-Uint64 CommandQueueVkImpl::ExecuteCommandBuffer(const VkSubmitInfo& SubmitInfo)
+Uint64 CommandQueueVkImpl::Submit(const VkSubmitInfo& SubmitInfo)
 {
     std::lock_guard<std::mutex> Lock(m_QueueMutex);
+    
+    auto FenceValue = m_NextFenceValue;
+    // Increment the value before submitting the buffer to be overly safe
+    Atomics::AtomicIncrement(m_NextFenceValue);
+
     auto vkFence = m_pFence->GetVkFence();
     bool SubmitCount = 
         (SubmitInfo.waitSemaphoreCount   != 0 || 
@@ -60,18 +64,15 @@ Uint64 CommandQueueVkImpl::ExecuteCommandBuffer(const VkSubmitInfo& SubmitInfo)
          SubmitInfo.signalSemaphoreCount != 0) ? 
         1 : 0;
     auto err = vkQueueSubmit(m_VkQueue, SubmitCount, &SubmitInfo, vkFence);
-    VERIFY(err == VK_SUCCESS, "Failed to submit command buffer to the command queue");
+    DEV_CHECK_ERR(err == VK_SUCCESS, "Failed to submit command buffer to the command queue");
 
     // We must atomically place the (value, fence) pair into the deque
-    auto FenceValue = m_NextFenceValue;
     m_pFence->AddPendingFence(std::move(vkFence), FenceValue);
 
-    // Increment the value
-    Atomics::AtomicIncrement(m_NextFenceValue);
     return FenceValue;
 }
 
-Uint64 CommandQueueVkImpl::ExecuteCommandBuffer(VkCommandBuffer cmdBuffer)
+Uint64 CommandQueueVkImpl::Submit(VkCommandBuffer cmdBuffer)
 {
     VkSubmitInfo SubmitInfo = {};
     SubmitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
@@ -87,10 +88,10 @@ Uint64 CommandQueueVkImpl::ExecuteCommandBuffer(VkCommandBuffer cmdBuffer)
     SubmitInfo.pSignalSemaphores = nullptr; // a pointer to an array of semaphores which will be signaled when the 
                                             // command buffers for this batch have completed execution
 
-    return ExecuteCommandBuffer(SubmitInfo);
+    return Submit(SubmitInfo);
 }
 
-Uint64 CommandQueueVkImpl::IdleGPU()
+Uint64 CommandQueueVkImpl::WaitForIdle()
 {
     std::lock_guard<std::mutex> Lock(m_QueueMutex);
 
@@ -113,8 +114,16 @@ Uint64 CommandQueueVkImpl::GetCompletedFenceValue()
 
 void CommandQueueVkImpl::SignalFence(VkFence vkFence)
 {
+    std::lock_guard<std::mutex> Lock(m_QueueMutex);
     auto err = vkQueueSubmit(m_VkQueue, 0, nullptr, vkFence);
-    VERIFY(err == VK_SUCCESS, "Failed to submit command buffer to the command queue");
+    DEV_CHECK_ERR(err == VK_SUCCESS, "Failed to submit command buffer to the command queue");
+}
+
+void CommandQueueVkImpl::Present(const VkPresentInfoKHR& PresentInfo)
+{
+    std::lock_guard<std::mutex> Lock(m_QueueMutex);
+    auto Result = vkQueuePresentKHR(m_VkQueue, &PresentInfo);
+    DEV_CHECK_ERR(Result == VK_SUCCESS, "Present failed");
 }
 
 }
diff --git a/Graphics/GraphicsEngineVulkan/src/DescriptorPoolManager.cpp b/Graphics/GraphicsEngineVulkan/src/DescriptorPoolManager.cpp
index 45cba04d..fe6e9c9f 100644
--- a/Graphics/GraphicsEngineVulkan/src/DescriptorPoolManager.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/DescriptorPoolManager.cpp
@@ -35,9 +35,9 @@ void DescriptorPoolAllocation::Release()
         VERIFY_EXPR(ParentPoolMgr != nullptr && ParentPool != nullptr);
         ParentPoolMgr->FreeAllocation(Set, *ParentPool);
 
-        Set = VK_NULL_HANDLE;
+        Set           = VK_NULL_HANDLE;
         ParentPoolMgr = nullptr;
-        ParentPool = nullptr;
+        ParentPool    = nullptr;
     }
 }
 
@@ -56,7 +56,7 @@ void DescriptorPoolManager::CreateNewPool()
     m_DescriptorPools.emplace_front( new VulkanUtilities::VulkanDescriptorPool(m_LogicalDevice, PoolCI) );
 }
 
-DescriptorPoolAllocation DescriptorPoolManager::Allocate(VkDescriptorSetLayout SetLayout)
+DescriptorPoolAllocation DescriptorPoolManager::Allocate(Uint64 CommandQueueMask, VkDescriptorSetLayout SetLayout)
 {
     // Descriptor pools are externally synchronized, meaning that the application must not allocate 
     // and/or free descriptor sets from the same pool in multiple threads simultaneously (13.2.3)
@@ -74,7 +74,7 @@ DescriptorPoolAllocation DescriptorPoolManager::Allocate(VkDescriptorSetLayout S
             {
                 std::swap(*it, m_DescriptorPools.front());
             }
-            return {Set, Pool, *this};
+            return {Set, CommandQueueMask, Pool, *this};
         }
     }
 
@@ -86,7 +86,7 @@ DescriptorPoolAllocation DescriptorPoolManager::Allocate(VkDescriptorSetLayout S
     auto Set = NewPool.AllocateDescriptorSet(SetLayout);
     VERIFY(Set != VK_NULL_HANDLE, "Failed to allocate descriptor set");
 
-    return {Set, NewPool, *this };
+    return {Set, CommandQueueMask, NewPool, *this };
 }
 
 void DescriptorPoolManager::FreeAllocation(VkDescriptorSet Set, VulkanUtilities::VulkanDescriptorPool& Pool)
diff --git a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
index 9be700a1..eb314ec4 100644
--- a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
@@ -63,16 +63,18 @@ namespace Diligent
                                               bool                                  bIsDeferred, 
                                               const EngineVkAttribs&                Attribs, 
                                               Uint32                                ContextId,
+                                              Uint32                                CommandQueueId,
                                               std::shared_ptr<GenerateMipsVkHelper> GenerateMipsHelper) :
         TDeviceContextBase{pRefCounters, pDeviceVkImpl, bIsDeferred},
         m_NumCommandsToFlush{bIsDeferred ? std::numeric_limits<decltype(m_NumCommandsToFlush)>::max() : Attribs.NumCommandsToFlushCmdBuffer},
         m_CmdListAllocator{ GetRawAllocator(), sizeof(CommandListVkImpl), 64 },
         m_ContextId{ContextId},
+        m_CommandQueueId{CommandQueueId},
         // Command pools for deferred contexts must be thread safe because finished command buffers are executed and released from another thread
         m_CmdPool
         {
             pDeviceVkImpl->GetLogicalDevice().GetSharedPtr(),
-            pDeviceVkImpl->GetCmdQueue()->GetQueueFamilyIndex(),
+            pDeviceVkImpl->GetCommandQueue(CommandQueueId).GetQueueFamilyIndex(),
             VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, 
             bIsDeferred
         },
@@ -149,12 +151,13 @@ namespace Diligent
         // call.
         VERIFY(m_UploadHeap.GetStaleAllocationsCount() == 0, "All stale allocations must have been discarded at this point");
         VERIFY(m_DynamicDescriptorPool.GetStaleAllocationCount() == 0, "All stale dynamic descriptor set allocations must have been discarded at this point");
-        ReleaseStaleContextResources(m_NextCmdBuffNumber, m_LastSubmittedFenceValue, pDeviceVkImpl->GetCompletedFenceValue());
+        // TODO: rework
+        ReleaseStaleContextResources(m_NextCmdBuffNumber, m_LastSubmittedFenceValue, pDeviceVkImpl->GetCompletedFenceValue(0));
         // Since we idled the GPU, all stale resources must have been destroyed now
         VERIFY(m_DynamicDescriptorPool.GetPendingReleaseAllocationCount() == 0, "All stale descriptor set allocations must have been destroyed at this point");
 
         auto VkCmdPool = m_CmdPool.Release();
-        pDeviceVkImpl->SafeReleaseVkObject(std::move(VkCmdPool));
+        pDeviceVkImpl->SafeReleaseDeviceObject(std::move(VkCmdPool), ~Uint64{0});
     }
 
     IMPLEMENT_QUERY_INTERFACE( DeviceContextVkImpl, IID_DeviceContextVk, TDeviceContextBase )
@@ -167,7 +170,8 @@ namespace Diligent
         if (m_CommandBuffer.GetVkCmdBuffer() == VK_NULL_HANDLE)
         {
             auto pDeviceVkImpl = m_pDevice.RawPtr<RenderDeviceVkImpl>();
-            auto vkCmdBuff = m_CmdPool.GetCommandBuffer(pDeviceVkImpl->GetCompletedFenceValue());
+            // TODO: rework
+            auto vkCmdBuff = m_CmdPool.GetCommandBuffer(pDeviceVkImpl->GetCompletedFenceValue(0));
             m_CommandBuffer.SetVkCmdBuffer(vkCmdBuff);
         }
     }
@@ -737,7 +741,8 @@ namespace Diligent
 
     void DeviceContextVkImpl::FinishFrame(bool ForceRelease)
     {
-        FinishFrame(ForceRelease ? std::numeric_limits<Uint64>::max() : m_pDevice.RawPtr<RenderDeviceVkImpl>()->GetCompletedFenceValue());
+        // TODO: rework
+        FinishFrame(ForceRelease ? std::numeric_limits<Uint64>::max() : m_pDevice.RawPtr<RenderDeviceVkImpl>()->GetCompletedFenceValue(0));
     }
 
     void DeviceContextVkImpl::FinishFrame(Uint64 CompletedFenceValue)
@@ -747,7 +752,7 @@ namespace Diligent
                 "There are outstanding commands in the deferred device context when finishing the frame. This is an error and may cause unpredicted behaviour. Close all deferred contexts and execute them before finishing the frame" :
                 "There are outstanding commands in the immediate device context when finishing the frame. This is an error and may cause unpredicted behaviour. Call Flush() to submit all commands for execution before finishing the frame");
 
-        m_UploadHeap.DiscardAllocations(m_LastSubmittedFenceValue);
+        m_UploadHeap.DiscardAllocations(m_CommandQueueId, m_LastSubmittedFenceValue);
         m_DynamicDescriptorPool.ReleaseStaleAllocations(CompletedFenceValue);
         m_DynamicHeap.FinishFrame(m_LastSubmittedFenceValue);
         Atomics::AtomicIncrement(m_ContextFrameNumber);
@@ -817,7 +822,8 @@ namespace Diligent
         // Release temporary resources that were used by this context while recording the last command buffer
         auto SubmittedCmdBuffNumber = m_NextCmdBuffNumber;
         Atomics::AtomicIncrement(m_NextCmdBuffNumber);
-        auto CompletedFenceValue = pDeviceVkImpl->GetCompletedFenceValue();
+        // TODO: rework
+        auto CompletedFenceValue = pDeviceVkImpl->GetCompletedFenceValue(0);
         ReleaseStaleContextResources(SubmittedCmdBuffNumber, m_LastSubmittedFenceValue, CompletedFenceValue);
 
         m_State = ContextState{};
@@ -1023,6 +1029,7 @@ namespace Diligent
 
             m_RenderPass = RPCache.GetRenderPass(RenderPassKey);
             FBKey.Pass = m_RenderPass;
+            FBKey.CommandQueueMask = ~Uint64{0};
             m_Framebuffer = FBCache.GetFramebuffer(FBKey, m_FramebufferWidth, m_FramebufferHeight, m_FramebufferSlices);
 
             // Set the viewport to match the render target size
@@ -1438,7 +1445,8 @@ namespace Diligent
         // record any commands and only need to add the buffer to the queue
         pDeferredCtxVkImpl->DisposeVkCmdBuffer(vkCmdBuff, SubmittedFenceValue);
         // We can now release all temporary resources in the deferred context associated with the submitted command list
-        auto CompletedFenceValue = pDeviceVkImpl->GetCompletedFenceValue();
+        // TODO: rework
+        auto CompletedFenceValue = pDeviceVkImpl->GetCompletedFenceValue(0);
         pDeferredCtxVkImpl->ReleaseStaleContextResources(DeferredCtxCmdBuffNumber, SubmittedFenceValue, CompletedFenceValue);
     }
 
diff --git a/Graphics/GraphicsEngineVulkan/src/FramebufferCache.cpp b/Graphics/GraphicsEngineVulkan/src/FramebufferCache.cpp
index a8a4a045..626420cb 100644
--- a/Graphics/GraphicsEngineVulkan/src/FramebufferCache.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/FramebufferCache.cpp
@@ -35,7 +35,8 @@ bool FramebufferCache::FramebufferCacheKey::operator == (const FramebufferCacheK
     if (GetHash()        != rhs.GetHash()        ||
         Pass             != rhs.Pass             ||
         NumRenderTargets != rhs.NumRenderTargets ||
-        DSV              != rhs.DSV)
+        DSV              != rhs.DSV              ||
+        CommandQueueMask != rhs.CommandQueueMask)
     {
         return false;
     }
@@ -51,7 +52,7 @@ size_t FramebufferCache::FramebufferCacheKey::GetHash()const
 {
     if (Hash == 0)
     {
-        Hash = ComputeHash(Pass, NumRenderTargets, DSV);
+        Hash = ComputeHash(Pass, NumRenderTargets, DSV, CommandQueueMask);
         for(Uint32 rt = 0; rt < NumRenderTargets; ++rt)
             HashCombine(Hash, RTVs[rt]);
     }
@@ -124,7 +125,7 @@ void FramebufferCache::OnDestroyImageView(VkImageView ImgView)
         // The framebuffer is deleted whenever any of the image views is deleted
         if (fb_it != m_Cache.end())
         {
-            m_DeviceVk.SafeReleaseVkObject(std::move(fb_it->second));
+            m_DeviceVk.SafeReleaseDeviceObject(std::move(fb_it->second), it->second.CommandQueueMask);
             m_Cache.erase(fb_it);
         }
     }
@@ -145,7 +146,7 @@ void FramebufferCache::OnDestroyRenderPass(VkRenderPass Pass)
         // The framebuffer is deleted whenever any of the image views or render pass is destroyed
         if (fb_it != m_Cache.end())
         {
-            m_DeviceVk.SafeReleaseVkObject(std::move(fb_it->second));
+            m_DeviceVk.SafeReleaseDeviceObject(std::move(fb_it->second), it->second.CommandQueueMask);
             m_Cache.erase(fb_it);
         }
     }
diff --git a/Graphics/GraphicsEngineVulkan/src/PipelineLayout.cpp b/Graphics/GraphicsEngineVulkan/src/PipelineLayout.cpp
index e34a9305..cd5c0c65 100644
--- a/Graphics/GraphicsEngineVulkan/src/PipelineLayout.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/PipelineLayout.cpp
@@ -183,9 +183,9 @@ void PipelineLayout::DescriptorSetLayoutManager::DescriptorSetLayout::Finalize(c
     pBindings = pNewBindings;
 }
 
-void PipelineLayout::DescriptorSetLayoutManager::DescriptorSetLayout::Release(RenderDeviceVkImpl *pRenderDeviceVk, IMemoryAllocator &MemAllocator)
+void PipelineLayout::DescriptorSetLayoutManager::DescriptorSetLayout::Release(RenderDeviceVkImpl *pRenderDeviceVk, IMemoryAllocator &MemAllocator, Uint64 CommandQueueMask)
 {
-    pRenderDeviceVk->SafeReleaseVkObject(std::move(VkLayout));
+    pRenderDeviceVk->SafeReleaseDeviceObject(std::move(VkLayout), CommandQueueMask);
     for (uint32_t b=0; b < NumLayoutBindings; ++b)
     {
         if (pBindings[b].pImmutableSamplers != nullptr)
@@ -276,12 +276,12 @@ void PipelineLayout::DescriptorSetLayoutManager::Finalize(const VulkanUtilities:
     VERIFY_EXPR(BindingOffset == TotalBindings);
 }
 
-void PipelineLayout::DescriptorSetLayoutManager::Release(RenderDeviceVkImpl *pRenderDeviceVk)
+void PipelineLayout::DescriptorSetLayoutManager::Release(RenderDeviceVkImpl *pRenderDeviceVk, Uint64 CommandQueueMask)
 {
     for (auto &Layout : m_DescriptorSetLayouts)
-        Layout.Release(pRenderDeviceVk, m_MemAllocator);
+        Layout.Release(pRenderDeviceVk, m_MemAllocator, CommandQueueMask);
 
-    pRenderDeviceVk->SafeReleaseVkObject(std::move(m_VkPipelineLayout));
+    pRenderDeviceVk->SafeReleaseDeviceObject(std::move(m_VkPipelineLayout), CommandQueueMask);
 }
 
 PipelineLayout::DescriptorSetLayoutManager::~DescriptorSetLayoutManager()
@@ -359,9 +359,9 @@ PipelineLayout::PipelineLayout() :
 {
 }
 
-void PipelineLayout::Release(RenderDeviceVkImpl *pDeviceVkImpl)
+void PipelineLayout::Release(RenderDeviceVkImpl *pDeviceVkImpl, Uint64 CommandQueueMask)
 {
-    m_LayoutMgr.Release(pDeviceVkImpl);
+    m_LayoutMgr.Release(pDeviceVkImpl, CommandQueueMask);
 }
 
 void PipelineLayout::AllocateResourceSlot(const SPIRVShaderResourceAttribs& ResAttribs,
@@ -416,7 +416,7 @@ void PipelineLayout::InitResourceCache(RenderDeviceVkImpl* pDeviceVkImpl, Shader
     const auto &StaticAndMutSet = m_LayoutMgr.GetDescriptorSet(SHADER_VARIABLE_TYPE_STATIC);
     if (StaticAndMutSet.SetIndex >= 0)
     {
-        DescriptorPoolAllocation SetAllocation = pDeviceVkImpl->AllocateDescriptorSet(StaticAndMutSet.VkLayout);
+        DescriptorPoolAllocation SetAllocation = pDeviceVkImpl->AllocateDescriptorSet(~Uint64{0}, StaticAndMutSet.VkLayout);
         ResourceCache.GetDescriptorSet(StaticAndMutSet.SetIndex).AssignDescriptorSetAllocation(std::move(SetAllocation));
     }
 }
diff --git a/Graphics/GraphicsEngineVulkan/src/PipelineStateVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/PipelineStateVkImpl.cpp
index f379d25f..d41983f0 100644
--- a/Graphics/GraphicsEngineVulkan/src/PipelineStateVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/PipelineStateVkImpl.cpp
@@ -402,14 +402,14 @@ PipelineStateVkImpl :: PipelineStateVkImpl(IReferenceCounters*      pRefCounters
 
 PipelineStateVkImpl::~PipelineStateVkImpl()
 {
-    m_pDevice->SafeReleaseVkObject(std::move(m_Pipeline));
-    m_PipelineLayout.Release(m_pDevice);
+    m_pDevice->SafeReleaseDeviceObject(std::move(m_Pipeline), m_Desc.CommandQueueMask);
+    m_PipelineLayout.Release(m_pDevice, m_Desc.CommandQueueMask);
 
     for (auto& ShaderModule : m_ShaderModules)
     {
         if (ShaderModule != VK_NULL_HANDLE)
         {
-            m_pDevice->SafeReleaseVkObject(std::move(ShaderModule));
+            m_pDevice->SafeReleaseDeviceObject(std::move(ShaderModule), m_Desc.CommandQueueMask);
         }
     }
 
diff --git a/Graphics/GraphicsEngineVulkan/src/RenderDeviceFactoryVk.cpp b/Graphics/GraphicsEngineVulkan/src/RenderDeviceFactoryVk.cpp
index 08ddd8bf..dcad3fbf 100644
--- a/Graphics/GraphicsEngineVulkan/src/RenderDeviceFactoryVk.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/RenderDeviceFactoryVk.cpp
@@ -251,7 +251,7 @@ void EngineFactoryVkImpl::AttachToVulkanDevice(std::shared_ptr<VulkanUtilities::
 
         std::shared_ptr<GenerateMipsVkHelper> GenerateMipsHelper(new GenerateMipsVkHelper(*pRenderDeviceVk));
 
-        RefCntAutoPtr<DeviceContextVkImpl> pImmediateCtxVk( NEW_RC_OBJ(RawMemAllocator, "DeviceContextVkImpl instance", DeviceContextVkImpl)(pRenderDeviceVk, false, EngineAttribs, 0, GenerateMipsHelper) );
+        RefCntAutoPtr<DeviceContextVkImpl> pImmediateCtxVk( NEW_RC_OBJ(RawMemAllocator, "DeviceContextVkImpl instance", DeviceContextVkImpl)(pRenderDeviceVk, false, EngineAttribs, 0, 0, GenerateMipsHelper) );
         // We must call AddRef() (implicitly through QueryInterface()) because pRenderDeviceVk will
         // keep a weak reference to the context
         pImmediateCtxVk->QueryInterface(IID_DeviceContext, reinterpret_cast<IObject**>(ppContexts) );
@@ -259,7 +259,7 @@ void EngineFactoryVkImpl::AttachToVulkanDevice(std::shared_ptr<VulkanUtilities::
 
         for (Uint32 DeferredCtx = 0; DeferredCtx < NumDeferredContexts; ++DeferredCtx)
         {
-            RefCntAutoPtr<DeviceContextVkImpl> pDeferredCtxVk( NEW_RC_OBJ(RawMemAllocator, "DeviceContextVkImpl instance", DeviceContextVkImpl)(pRenderDeviceVk, true, EngineAttribs, 1+DeferredCtx, GenerateMipsHelper) );
+            RefCntAutoPtr<DeviceContextVkImpl> pDeferredCtxVk( NEW_RC_OBJ(RawMemAllocator, "DeviceContextVkImpl instance", DeviceContextVkImpl)(pRenderDeviceVk, true, EngineAttribs, 1+DeferredCtx, 0, GenerateMipsHelper) );
             // We must call AddRef() (implicitly through QueryInterface()) because pRenderDeviceVk will
             // keep a weak reference to the context
             pDeferredCtxVk->QueryInterface(IID_DeviceContext, reinterpret_cast<IObject**>(ppContexts + 1 + DeferredCtx) );
diff --git a/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp
index b6ab45bf..4da6e760 100644
--- a/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp
@@ -50,6 +50,7 @@ RenderDeviceVkImpl :: RenderDeviceVkImpl(IReferenceCounters*
     {
         pRefCounters,
         RawMemAllocator,
+        1,
         NumDeferredContexts,
         sizeof(TextureVkImpl),
         sizeof(TextureViewVkImpl),
@@ -64,10 +65,7 @@ RenderDeviceVkImpl :: RenderDeviceVkImpl(IReferenceCounters*
     m_VulkanInstance(Instance),
     m_PhysicalDevice(std::move(PhysicalDevice)),
     m_LogicalVkDevice(std::move(LogicalDevice)),
-    m_pCommandQueue(pCmdQueue),
     m_EngineAttribs(CreationAttribs),
-	m_FrameNumber(0),
-    m_NextCmdBuffNumber(0),
     m_FramebufferCache(*this),
     m_RenderPassCache(*this),
     m_MainDescriptorPool
@@ -90,14 +88,16 @@ RenderDeviceVkImpl :: RenderDeviceVkImpl(IReferenceCounters*
     },
     m_TransientCmdPoolMgr(*m_LogicalVkDevice, pCmdQueue->GetQueueFamilyIndex(), VK_COMMAND_POOL_CREATE_TRANSIENT_BIT),
     m_MemoryMgr("Global resource memory manager", *m_LogicalVkDevice, *m_PhysicalDevice, GetRawAllocator(), CreationAttribs.DeviceLocalMemoryPageSize, CreationAttribs.HostVisibleMemoryPageSize, CreationAttribs.DeviceLocalMemoryReserveSize, CreationAttribs.HostVisibleMemoryReserveSize),
-    m_ReleaseQueue(GetRawAllocator()),
     m_DynamicMemoryManager
     {
         GetRawAllocator(),
         *this,
-        CreationAttribs.DynamicHeapSize
+        CreationAttribs.DynamicHeapSize,
+        ~Uint64{0}
     }
 {
+    m_CommandQueues[0].CmdQueue = pCmdQueue;
+
     m_DeviceCaps.DevType = DeviceType::Vulkan;
     m_DeviceCaps.MajorVersion = 1;
     m_DeviceCaps.MinorVersion = 0;
@@ -121,7 +121,10 @@ RenderDeviceVkImpl::~RenderDeviceVkImpl()
     // release queues
     FinishFrame(true);
 
-    m_TransientCmdPoolMgr.DestroyPools(m_pCommandQueue->GetCompletedFenceValue());
+    m_TransientCmdPoolMgr.DestroyPools(m_CommandQueues[0].CmdQueue->GetCompletedFenceValue());
+
+    // We must destroy command queues explicitly prior to releasing Vulkan device
+    DestroyCommandQueues();
 
     //if(m_PhysicalDevice)
     //{
@@ -134,7 +137,8 @@ RenderDeviceVkImpl::~RenderDeviceVkImpl()
 
 void RenderDeviceVkImpl::AllocateTransientCmdPool(VulkanUtilities::CommandPoolWrapper& CmdPool, VkCommandBuffer& vkCmdBuff, const Char *DebugPoolName)
 {
-    auto CompletedFenceValue = GetCompletedFenceValue();
+    // TODO: rework this
+    auto CompletedFenceValue = m_CommandQueues[0].CmdQueue->GetCompletedFenceValue();
     CmdPool = m_TransientCmdPoolMgr.AllocateCommandPool(CompletedFenceValue, DebugPoolName);
 
     // Allocate command buffer from the cmd pool
@@ -158,7 +162,7 @@ void RenderDeviceVkImpl::AllocateTransientCmdPool(VulkanUtilities::CommandPoolWr
 }
 
 
-void RenderDeviceVkImpl::ExecuteAndDisposeTransientCmdBuff(VkCommandBuffer vkCmdBuff, VulkanUtilities::CommandPoolWrapper&& CmdPool)
+void RenderDeviceVkImpl::ExecuteAndDisposeTransientCmdBuff(Uint32 QueueIndex, VkCommandBuffer vkCmdBuff, VulkanUtilities::CommandPoolWrapper&& CmdPool)
 {
     VERIFY_EXPR(vkCmdBuff != VK_NULL_HANDLE);
 
@@ -170,11 +174,6 @@ void RenderDeviceVkImpl::ExecuteAndDisposeTransientCmdBuff(VkCommandBuffer vkCmd
     SubmitInfo.commandBufferCount = 1;
     SubmitInfo.pCommandBuffers = &vkCmdBuff;
 
-    Uint64 SubmittedFenceValue = 0;
-    Uint64 SubmittedCmdBuffNumber = 0;
-    SubmitCommandBuffer(SubmitInfo, SubmittedCmdBuffNumber, SubmittedFenceValue, nullptr);
-
-
     // We MUST NOT discard stale objects when executing transient command buffer,
     // otherwise a resource can be destroyed while still being used by the GPU:
     // 
@@ -197,10 +196,12 @@ void RenderDeviceVkImpl::ExecuteAndDisposeTransientCmdBuff(VkCommandBuffer vkCmd
     //              |            |  - ResourceX is already in release     |                                      |
     //              |            |    queue with fence value F, and       |                                      |
     //              |            |    F < SubmittedFenceValue==F+1        |                                      |
-    
-
+    //
+    // Since transient command buffers do not count as real command buffers, submit them directly to the queue
+    // to avoid interference with the command buffer numbers
+    Uint64 FenceValue = GetCommandQueue(0).Submit(SubmitInfo);
     // Dispose command pool
-    m_TransientCmdPoolMgr.DisposeCommandPool(std::move(CmdPool), SubmittedFenceValue);
+    m_TransientCmdPoolMgr.DisposeCommandPool(std::move(CmdPool), FenceValue);
 }
 
 void RenderDeviceVkImpl::SubmitCommandBuffer(const VkSubmitInfo& SubmitInfo, 
@@ -209,22 +210,18 @@ void RenderDeviceVkImpl::SubmitCommandBuffer(const VkSubmitInfo& SubmitInfo,
                                              std::vector<std::pair<Uint64, RefCntAutoPtr<IFence> > >* pFences // List of fences to signal
                                              )
 {
-    // The lock is required to atomically update m_NextCmdBuffNumber
-	std::lock_guard<std::mutex> LockGuard(m_SubmitCmdBufferMutex);
-    auto NextFenceValue = m_pCommandQueue->GetNextFenceValue();
 	// Submit the command list to the queue
-    SubmittedFenceValue = m_pCommandQueue->ExecuteCommandBuffer(SubmitInfo);
-    VERIFY(SubmittedFenceValue >= NextFenceValue, "Fence value of the executed command list is less than the next fence value previously queried through GetNextFenceValue()");
-    SubmittedFenceValue = std::max(SubmittedFenceValue, NextFenceValue);
-    SubmittedCmdBuffNumber = m_NextCmdBuffNumber;
-    Atomics::AtomicIncrement(m_NextCmdBuffNumber);
+    Uint32 QueueIndex = 0;
+    auto CmbBuffInfo = TRenderDeviceBase::SubmitCommandBuffer(QueueIndex, SubmitInfo, true);
+    SubmittedFenceValue    = CmbBuffInfo.FenceValue;
+    SubmittedCmdBuffNumber = CmbBuffInfo.CmdBufferNumber;
     if (pFences != nullptr)
     {
         for (auto& val_fence : *pFences)
         {
             auto* pFenceVkImpl = val_fence.second.RawPtr<FenceVkImpl>();
             auto vkFence = pFenceVkImpl->GetVkFence();
-            m_pCommandQueue->SignalFence(vkFence);
+            m_CommandQueues[QueueIndex].CmdQueue->SignalFence(vkFence);
             pFenceVkImpl->AddPendingFence(std::move(vkFence), val_fence.first);
         }
     }
@@ -240,44 +237,20 @@ Uint64 RenderDeviceVkImpl::ExecuteCommandBuffer(const VkSubmitInfo& SubmitInfo,
     Uint64 SubmittedCmdBuffNumber = 0;
     SubmitCommandBuffer(SubmitInfo, SubmittedCmdBuffNumber, SubmittedFenceValue, pSignalFences);
 
-    // The following basic requirement guarantees correctness of resource deallocation:
-    //
-    //        A resource is never released before the last draw command referencing it is invoked on the immediate context
-    //
-
-    // Move stale objects into the release queue.
-    // Note that objects are moved from stale list to release queue based on the cmd buffer number, 
-    // not fence value. This makes sure that basic requirement is met even when the fence value is 
-    // not incremented while executing the command buffer (as is the case with Unity command queue).
-
-    // As long as resources used by deferred contexts are not released until the command list
-    // is executed through immediate context, this stategy always works.
-
-    auto CompletedFenceValue = GetCompletedFenceValue();
-    ProcessStaleResources(SubmittedCmdBuffNumber, SubmittedFenceValue, CompletedFenceValue);
+    // TODO: rework this
+    auto CompletedFenceValue = m_CommandQueues[0].CmdQueue->GetCompletedFenceValue();
+    m_MainDescriptorPool.ReleaseStaleAllocations(CompletedFenceValue);
+    m_MemoryMgr.ShrinkMemory();
+    PurgeReleaseQueues();
 
     return SubmittedFenceValue;
 }
 
 
-Uint64 RenderDeviceVkImpl::IdleGPU(bool ReleaseStaleObjects) 
+void RenderDeviceVkImpl::IdleGPU(bool ReleaseStaleObjects) 
 { 
-    Uint64 SubmittedFenceValue = 0;
-    Uint64 SubmittedCmdBuffNumber = 0;
-
-    {
-        // CommandQueueVkImpl::IdleGPU increments next fence value and returns
-        // the previous value
-        SubmittedFenceValue = m_pCommandQueue->IdleGPU();
-
-        m_LogicalVkDevice->WaitIdle();
-
-        // Increment cmd list number while keeping queue locked. 
-        // This guarantees that any Vk object released after the lock
-        // is released, will be associated with the incremented cmd list number
-        SubmittedCmdBuffNumber = m_NextCmdBuffNumber;
-        Atomics::AtomicIncrement(m_NextCmdBuffNumber);
-    }
+    IdleCommandQueues(ReleaseStaleObjects, ReleaseStaleObjects);
+    m_LogicalVkDevice->WaitIdle();
 
     if (ReleaseStaleObjects)
     {
@@ -287,54 +260,33 @@ Uint64 RenderDeviceVkImpl::IdleGPU(bool ReleaseStaleObjects)
         // Since GPU has been idled, it it is safe to do so
 
         // SubmittedFenceValue has now been signaled by the GPU since we waited for it
-        auto CompletedFenceValue = SubmittedFenceValue;
-        ProcessStaleResources(SubmittedCmdBuffNumber, SubmittedFenceValue, CompletedFenceValue);
+        m_MainDescriptorPool.ReleaseStaleAllocations(m_CommandQueues[0].CmdQueue->GetCompletedFenceValue());
+        m_MemoryMgr.ShrinkMemory();
     }
-
-    return SubmittedFenceValue;
-}
-
-
-Bool RenderDeviceVkImpl::IsFenceSignaled(Uint64 FenceValue) 
-{
-    return FenceValue <= GetCompletedFenceValue();
-}
-
-Uint64 RenderDeviceVkImpl::GetCompletedFenceValue()
-{
-    return m_pCommandQueue->GetCompletedFenceValue();
 }
 
 
 void RenderDeviceVkImpl::FinishFrame(bool ReleaseAllResources)
 {
-    auto CompletedFenceValue = ReleaseAllResources ? std::numeric_limits<Uint64>::max() : GetCompletedFenceValue();
-
-
+    // TODO: rework this
+    auto CompletedFenceValue = ReleaseAllResources ? std::numeric_limits<Uint64>::max() : m_CommandQueues[0].CmdQueue->GetCompletedFenceValue();
+    
+    // Discard all remaining objects. This is important to do if there were 
+    // no command lists submitted during the frame. All stale resources will
+    // be associated with the submitted fence value and thus will not be released
+    // until the GPU is finished with the current frame
     Uint64 SubmittedFenceValue = 0;
     Uint64 SubmittedCmdBuffNumber = 0;
     VkSubmitInfo DummySubmitInfo = {};
     // Submit empty command buffer to set a fence on the GPU
     SubmitCommandBuffer(DummySubmitInfo, SubmittedCmdBuffNumber, SubmittedFenceValue, nullptr);
         
-    // Discard all remaining objects. This is important to do if there were 
-    // no command lists submitted during the frame. All stale resources will
-    // be associated with the submitted fence value and thus will not be released
-    // until the GPU is finished with the current frame
-    ProcessStaleResources(SubmittedCmdBuffNumber, SubmittedFenceValue, CompletedFenceValue);
-
-    m_DynamicMemoryManager.ReleaseStaleBlocks(CompletedFenceValue);
+    PurgeReleaseQueues();
 
-    Atomics::AtomicIncrement(m_FrameNumber);
-}
-
-
-void RenderDeviceVkImpl::ProcessStaleResources(Uint64 SubmittedCmdBufferNumber, Uint64 SubmittedFenceValue, Uint64 CompletedFenceValue)
-{
-    m_ReleaseQueue.DiscardStaleResources(SubmittedCmdBufferNumber, SubmittedFenceValue);
-    m_ReleaseQueue.Purge(CompletedFenceValue);
     m_MainDescriptorPool.ReleaseStaleAllocations(CompletedFenceValue);
     m_MemoryMgr.ShrinkMemory();
+
+    m_DynamicMemoryManager.ReleaseStaleBlocks(CompletedFenceValue);
 }
 
 
diff --git a/Graphics/GraphicsEngineVulkan/src/SamplerVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/SamplerVkImpl.cpp
index 8f748100..292c4fea 100644
--- a/Graphics/GraphicsEngineVulkan/src/SamplerVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/SamplerVkImpl.cpp
@@ -75,7 +75,7 @@ SamplerVkImpl::SamplerVkImpl(IReferenceCounters* pRefCounters, RenderDeviceVkImp
 
 SamplerVkImpl::~SamplerVkImpl()
 {
-    m_pDevice->SafeReleaseVkObject(std::move(m_VkSampler));
+    m_pDevice->SafeReleaseDeviceObject(std::move(m_VkSampler), m_CommandQueueMask);
 }
 
 IMPLEMENT_QUERY_INTERFACE( SamplerVkImpl, IID_SamplerVk, TSamplerBase )
diff --git a/Graphics/GraphicsEngineVulkan/src/SwapChainVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/SwapChainVkImpl.cpp
index 3f59f2ee..930951f6 100644
--- a/Graphics/GraphicsEngineVulkan/src/SwapChainVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/SwapChainVkImpl.cpp
@@ -84,8 +84,8 @@ SwapChainVkImpl::SwapChainVkImpl(IReferenceCounters*    pRefCounters,
 
     CHECK_VK_ERROR_AND_THROW(err, "Failed to create OS-specific surface");
     const auto& PhysicalDevice = pRenderDeviceVk->GetPhysicalDevice();
-    auto *CmdQueueVK = pRenderDeviceVk->GetCmdQueue();
-    auto QueueFamilyIndex = CmdQueueVK->GetQueueFamilyIndex();
+    auto& CmdQueueVK = pRenderDeviceVk->GetCommandQueue(0);
+    auto QueueFamilyIndex = CmdQueueVK.GetQueueFamilyIndex();
     if( !PhysicalDevice.CheckPresentSupport(QueueFamilyIndex, m_VkSurface) )
     {
         LOG_ERROR_AND_THROW("Selected physical device does not support present capability.\n"
@@ -454,10 +454,7 @@ void SwapChainVkImpl::Present(Uint32 SyncInterval)
         PresentInfo.pImageIndices = &m_BackBufferIndex;
         VkResult Result = VK_SUCCESS;
         PresentInfo.pResults = &Result;
-    
-        auto vkCmdQueue = pDeviceVk->GetCmdQueue()->GetVkQueue();
-        vkQueuePresentKHR(vkCmdQueue, &PresentInfo);
-        VERIFY(Result == VK_SUCCESS, "Present failed");
+        pDeviceVk->GetCommandQueue(0).Present(PresentInfo);
     }
 
     pImmediateCtxVk->FinishFrame(false);
diff --git a/Graphics/GraphicsEngineVulkan/src/TextureViewVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/TextureViewVkImpl.cpp
index 72b6b34b..3ec73fe1 100644
--- a/Graphics/GraphicsEngineVulkan/src/TextureViewVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/TextureViewVkImpl.cpp
@@ -44,7 +44,7 @@ TextureViewVkImpl::~TextureViewVkImpl()
 {
     if(m_Desc.ViewType == TEXTURE_VIEW_DEPTH_STENCIL || m_Desc.ViewType == TEXTURE_VIEW_RENDER_TARGET)
         m_pDevice->GetFramebufferCache().OnDestroyImageView(m_ImageView);
-    m_pDevice->SafeReleaseVkObject(std::move(m_ImageView));
+    m_pDevice->SafeReleaseDeviceObject(std::move(m_ImageView), m_pTexture->GetDesc().CommandQueueMask);
 }
 
 IMPLEMENT_QUERY_INTERFACE( TextureViewVkImpl, IID_TextureViewVk, TTextureViewBase )
diff --git a/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp
index b95094f4..4562a073 100644
--- a/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp
@@ -335,13 +335,14 @@ TextureVkImpl :: TextureVkImpl(IReferenceCounters*          pRefCounters,
             m_CurrentLayout, // dstImageLayout must be VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL or VK_IMAGE_LAYOUT_GENERAL (18.4)
             static_cast<uint32_t>(Regions.size()), Regions.data());
 
-	    pRenderDeviceVk->ExecuteAndDisposeTransientCmdBuff(vkCmdBuff, std::move(CmdPool));
+        Uint32 QueueIndex = 0;
+	    pRenderDeviceVk->ExecuteAndDisposeTransientCmdBuff(QueueIndex, vkCmdBuff, std::move(CmdPool));
 
         // After command buffer is submitted, safe-release resources. This strategy
         // is little overconservative as the resources will be released after the first
         // command buffer submitted through the immediate context will be completed
-        pRenderDeviceVk->SafeReleaseVkObject(std::move(StagingBuffer));
-        pRenderDeviceVk->SafeReleaseVkObject(std::move(StagingMemoryAllocation));
+        pRenderDeviceVk->SafeReleaseDeviceObject(std::move(StagingBuffer),           Uint64{1} << Uint64{QueueIndex});
+        pRenderDeviceVk->SafeReleaseDeviceObject(std::move(StagingMemoryAllocation), Uint64{1} << Uint64{QueueIndex});
     }
     else
     {
@@ -373,7 +374,8 @@ TextureVkImpl :: TextureVkImpl(IReferenceCounters*          pRefCounters,
         {
             UNEXPECTED("Unexpected aspect mask");
         }
-        pRenderDeviceVk->ExecuteAndDisposeTransientCmdBuff(vkCmdBuff, std::move(CmdPool));
+        Uint32 QueueIndex = 0;
+        pRenderDeviceVk->ExecuteAndDisposeTransientCmdBuff(QueueIndex, vkCmdBuff, std::move(CmdPool));
     }
 
 
@@ -480,8 +482,8 @@ TextureVkImpl :: ~TextureVkImpl()
 {
     // Vk object can only be destroyed when it is no longer used by the GPU
     // Wrappers for external texture will not be destroyed as they are created with null device pointer
-    m_pDevice->SafeReleaseVkObject(std::move(m_VulkanImage));
-    m_pDevice->SafeReleaseVkObject(std::move(m_MemoryAllocation));
+    m_pDevice->SafeReleaseDeviceObject(std::move(m_VulkanImage),      m_Desc.CommandQueueMask);
+    m_pDevice->SafeReleaseDeviceObject(std::move(m_MemoryAllocation), m_Desc.CommandQueueMask);
 }
 
 void TextureVkImpl::UpdateData( IDeviceContext *pContext, Uint32 MipLevel, Uint32 Slice, const Box& DstBox, const TextureSubResData& SubresData )
diff --git a/Graphics/GraphicsEngineVulkan/src/VulkanDynamicHeap.cpp b/Graphics/GraphicsEngineVulkan/src/VulkanDynamicHeap.cpp
index e535dcad..90e585ba 100644
--- a/Graphics/GraphicsEngineVulkan/src/VulkanDynamicHeap.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/VulkanDynamicHeap.cpp
@@ -39,10 +39,12 @@ static VkDeviceSize GetDefaultAlignment(const VulkanUtilities::VulkanPhysicalDev
 
 VulkanDynamicMemoryManager::VulkanDynamicMemoryManager(IMemoryAllocator&   Allocator, 
                                                        RenderDeviceVkImpl& DeviceVk, 
-                                                       Uint32              Size) :
+                                                       Uint32              Size,
+                                                       Uint64              CommandQueueMask) :
     TBase(Allocator, Size),
     m_DeviceVk(DeviceVk),
-    m_DefaultAlignment(GetDefaultAlignment(DeviceVk.GetPhysicalDevice()))
+    m_DefaultAlignment(GetDefaultAlignment(DeviceVk.GetPhysicalDevice())),
+    m_CommandQueueMask(CommandQueueMask)
 {
     VERIFY( (Size & (MasterBlockAlignment-1)) == 0, "Heap size (", Size, " is not aligned by the master block alignment (", MasterBlockAlignment, ")");
     VkBufferCreateInfo VkBuffCI = {};
@@ -105,8 +107,8 @@ void VulkanDynamicMemoryManager::Destroy()
     if (m_VkBuffer)
     {
         m_DeviceVk.GetLogicalDevice().UnmapMemory(m_BufferMemory);
-        m_DeviceVk.SafeReleaseVkObject(std::move(m_VkBuffer));
-        m_DeviceVk.SafeReleaseVkObject(std::move(m_BufferMemory));
+        m_DeviceVk.SafeReleaseDeviceObject(std::move(m_VkBuffer),     m_CommandQueueMask);
+        m_DeviceVk.SafeReleaseDeviceObject(std::move(m_BufferMemory), m_CommandQueueMask);
     }
     m_CPUAddress = nullptr;
 }
@@ -144,7 +146,8 @@ VulkanDynamicMemoryManager::MasterBlock VulkanDynamicMemoryManager::AllocateMast
         Uint32 SleepIterations = 0;
         while (Block.UnalignedOffset == InvalidOffset && IdleDuration < MaxIdleDuration)
         {
-            auto LastCompletedFenceValue = m_DeviceVk.GetCompletedFenceValue();
+            // TODO: rework
+            auto LastCompletedFenceValue = m_DeviceVk.GetCompletedFenceValue(0);
             ReleaseStaleBlocks(LastCompletedFenceValue);
             Block = AllocateMasterBlock(SizeInBytes, Alignment);
             if (Block.UnalignedOffset == InvalidOffset)
diff --git a/Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp b/Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp
index a41a010f..4dd0caa0 100644
--- a/Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp
@@ -126,9 +126,9 @@ VulkanUploadAllocation VulkanUploadHeap::Allocate(size_t SizeInBytes, size_t Ali
     return Allocation;
 }
 
-void VulkanUploadHeap::DiscardAllocations(uint64_t FenceValue)
+void VulkanUploadHeap::DiscardAllocations(Uint32 CommandQueueIndex, uint64_t FenceValue)
 {
-    auto& ReleaseQueue = m_RenderDevice.GetReleaseQueue();
+    auto& ReleaseQueue = m_RenderDevice.GetReleaseQueue(CommandQueueIndex);
 
     {
         auto AllocIt = m_Pages.begin();