Unified resource liftime management in D3D12 and Vk backends

14 files changed, 133 insertions, 193 deletions

diff --git a/Graphics/GraphicsEngineD3D12/include/CommandQueueD3D12Impl.h b/Graphics/GraphicsEngineD3D12/include/CommandQueueD3D12Impl.h
index 192210f4..fa542481 100644
--- a/Graphics/GraphicsEngineD3D12/include/CommandQueueD3D12Impl.h
+++ b/Graphics/GraphicsEngineD3D12/include/CommandQueueD3D12Impl.h
@@ -28,6 +28,7 @@
 
 #include "CommandQueueD3D12.h"
 #include "ObjectBase.h"
+#include <mutex>
 
 namespace Diligent
 {
@@ -44,14 +45,14 @@ public:
     virtual void QueryInterface( const Diligent::INTERFACE_ID &IID, IObject **ppInterface )override;
 
 	// Returns the fence value that will be signaled next time
-    virtual UINT64 GetNextFenceValue()override final { return m_NextFenceValue; }
+    virtual Uint64 GetNextFenceValue()override final { return m_NextFenceValue; }
 
 	// Executes a given command list
-	virtual UINT64 ExecuteCommandList(ID3D12GraphicsCommandList* commandList)override final;
+	virtual Uint64 Submit(ID3D12GraphicsCommandList* commandList)override final;
 
     virtual ID3D12CommandQueue* GetD3D12CommandQueue()override final { return m_pd3d12CmdQueue; }
 
-    virtual void IdleGPU()override final;
+    virtual Uint64 WaitForIdle()override final;
 
     virtual Uint64 GetCompletedFenceValue()override final;
 
@@ -64,6 +65,7 @@ private:
     // Last fence value completed by the GPU
     volatile Uint64 m_LastCompletedFenceValue = 0;
 
+    std::mutex                  m_QueueMtx;
     CComPtr<ID3D12CommandQueue> m_pd3d12CmdQueue;
 
     // The fence is signaled right after the command list has been 
diff --git a/Graphics/GraphicsEngineD3D12/include/RenderDeviceD3D12Impl.h b/Graphics/GraphicsEngineD3D12/include/RenderDeviceD3D12Impl.h
index 0baaea26..a9bcaf70 100644
--- a/Graphics/GraphicsEngineD3D12/include/RenderDeviceD3D12Impl.h
+++ b/Graphics/GraphicsEngineD3D12/include/RenderDeviceD3D12Impl.h
@@ -35,7 +35,6 @@
 #include "D3D12DynamicHeap.h"
 #include "Atomics.h"
 #include "CommandQueueD3D12.h"
-#include "ResourceReleaseQueue.h"
 
 /// Namespace for the Direct3D11 implementation of the graphics engine
 namespace Diligent
@@ -51,7 +50,8 @@ public:
                            IMemoryAllocator&         RawMemAllocator, 
                            const EngineD3D12Attribs& CreationAttribs, 
                            ID3D12Device*             pD3D12Device, 
-                           ICommandQueueD3D12*       pCmdQueue, 
+                           size_t                    CommandQueueCount,
+                           ICommandQueueD3D12**      ppCmdQueues, 
                            Uint32                    NumDeferredContexts );
     ~RenderDeviceD3D12Impl();
 
@@ -80,23 +80,12 @@ public:
     DescriptorHeapAllocation AllocateDescriptor( D3D12_DESCRIPTOR_HEAP_TYPE Type, UINT Count = 1 );
     DescriptorHeapAllocation AllocateGPUDescriptors( D3D12_DESCRIPTOR_HEAP_TYPE Type, UINT Count = 1 );
 
-    Uint64 GetCompletedFenceValue();
-	virtual Uint64 GetNextFenceValue() override final
-    {
-        return m_pCommandQueue->GetNextFenceValue();
-    }
-
-	Uint64 GetCurrentFrameNumber()const {return static_cast<Uint64>(m_FrameNumber);}
-    virtual Bool IsFenceSignaled(Uint64 FenceValue) override final;
-
-    ICommandQueueD3D12 *GetCmdQueue(){return m_pCommandQueue;}
-
 	void IdleGPU(bool ReleaseStaleObjects);
     CommandContext* AllocateCommandContext(const Char *ID = "");
+    void CloseAndExecuteTransientCommandContext(Uint32 CommandQueueIndex, CommandContext *pCtx);
     Uint64 CloseAndExecuteCommandContext(CommandContext *pCtx, bool DiscardStaleObjects, std::vector<std::pair<Uint64, RefCntAutoPtr<IFence> > >* pSignalFences);
     void DisposeCommandContext(CommandContext*);
 
-    void SafeReleaseD3D12Object(ID3D12Object* pObj);
     void FinishFrame(bool ReleaseAllResources);
     virtual void FinishFrame()override final { FinishFrame(false); }
 
@@ -107,7 +96,6 @@ private:
 
     /// D3D12 device
     CComPtr<ID3D12Device> m_pd3d12Device;
-    RefCntAutoPtr<ICommandQueueD3D12> m_pCommandQueue;
 
     EngineD3D12Attribs m_EngineAttribs;
 
@@ -117,39 +105,6 @@ private:
 	
 	const Uint32 m_DynamicDescriptorAllocationChunkSize[2];
 
-	std::mutex m_CmdQueueMutex;
-
-	Atomics::AtomicInt64 m_FrameNumber;
-    Atomics::AtomicInt64 m_NextCmdListNumber;
-
-    // The following basic requirement guarantees correctness of resource deallocation:
-    //
-    //        A resource is never released before the last draw command referencing it is invoked on the immediate context
-    //
-    // See http://diligentgraphics.com/diligent-engine/architecture/d3d12/managing-resource-lifetimes/
-
-    //
-    // CPU
-    //                       Last Reference
-    //                        of resource X
-    //                             |
-    //                             |     Submit Cmd       Submit Cmd            Submit Cmd
-    //                             |      List N           List N+1              List N+2
-    //                             V         |                |                     |
-    //    NextFenceValue       |   *  N      |      N+1       |          N+2        |
-    //
-    //
-    //    CompletedFenceValue       |     N-3      |      N-2      |        N-1        |        N       |
-    //                              .              .               .                   .                .
-    // -----------------------------.--------------.---------------.-------------------.----------------.-------------
-    //                              .              .               .                   .                .
-    //       
-    // GPU                          | Cmd List N-2 | Cmd List N-1  |    Cmd List N     |   Cmd List N+1 |
-    //                                                                                 |
-    //                                                                                 |
-    //                                                                          Resource X can
-    //                                                                           be released
-
     CommandListManager m_CmdListManager;
 
     typedef std::unique_ptr<CommandContext, STDDeleterRawMem<CommandContext> > ContextPoolElemType;
@@ -158,8 +113,6 @@ private:
 	std::deque<CommandContext*, STDAllocatorRawMem<CommandContext*> > m_AvailableContexts;
 	std::mutex m_ContextAllocationMutex;
 
-    ResourceReleaseQueue<StaticStaleResourceWrapper<CComPtr<ID3D12Object>>> m_ReleaseQueue;
-
     D3D12DynamicMemoryManager m_DynamicMemoryManager;
 };
 
diff --git a/Graphics/GraphicsEngineD3D12/interface/CommandQueueD3D12.h b/Graphics/GraphicsEngineD3D12/interface/CommandQueueD3D12.h
index 62aaf3b3..ccc22374 100644
--- a/Graphics/GraphicsEngineD3D12/interface/CommandQueueD3D12.h
+++ b/Graphics/GraphicsEngineD3D12/interface/CommandQueueD3D12.h
@@ -38,12 +38,12 @@ class ICommandQueueD3D12 : public Diligent::IObject
 {
 public:
 	/// Returns the fence value that will be signaled next time
-	virtual UINT64 GetNextFenceValue() = 0;
+	virtual Uint64 GetNextFenceValue() = 0;
 
 	/// Executes a given command list
 
     /// \return Fence value associated with the executed command list
-	virtual UINT64 ExecuteCommandList(ID3D12GraphicsCommandList* commandList) = 0;
+	virtual Uint64 Submit(ID3D12GraphicsCommandList* commandList) = 0;
 
     /// Returns D3D12 command queue. May return null if queue is anavailable
     virtual ID3D12CommandQueue* GetD3D12CommandQueue() = 0;
@@ -52,7 +52,7 @@ public:
     virtual Uint64 GetCompletedFenceValue() = 0;
 
     /// Blocks execution until all pending GPU commands are complete
-    virtual void IdleGPU() = 0;
+    virtual Uint64 WaitForIdle() = 0;
 
     /// Signals the given fence
     virtual void SignalFence(ID3D12Fence* pFence, Uint64 Value) = 0;
diff --git a/Graphics/GraphicsEngineD3D12/interface/RenderDeviceD3D12.h b/Graphics/GraphicsEngineD3D12/interface/RenderDeviceD3D12.h
index 70d111cb..1c97374c 100644
--- a/Graphics/GraphicsEngineD3D12/interface/RenderDeviceD3D12.h
+++ b/Graphics/GraphicsEngineD3D12/interface/RenderDeviceD3D12.h
@@ -46,11 +46,14 @@ public:
     virtual ID3D12Device* GetD3D12Device() = 0;
 
     /// Returns the fence value that will be signaled by the GPU command queue next
-    virtual Uint64 GetNextFenceValue() = 0;
+    virtual Uint64 GetNextFenceValue(Uint32 QueueIndex) = 0;
+
+    /// Returns the last completed fence value for the given command queue
+    virtual Uint64 GetCompletedFenceValue(Uint32 QueueIndex) = 0;
 
     /// Checks if the fence value has been signaled by the GPU. True means
     /// that all associated work has been finished
-    virtual Bool IsFenceSignaled(Uint64 FenceValue) = 0;
+    virtual Bool IsFenceSignaled(Uint32 QueueIndex, Uint64 FenceValue) = 0;
 
     /// Should be called at the end of the frame when attached to existing D3D12 device
     /// Otherwise the method is automatically called before present
diff --git a/Graphics/GraphicsEngineD3D12/interface/RenderDeviceFactoryD3D12.h b/Graphics/GraphicsEngineD3D12/interface/RenderDeviceFactoryD3D12.h
index 67d994c4..5b1a741e 100644
--- a/Graphics/GraphicsEngineD3D12/interface/RenderDeviceFactoryD3D12.h
+++ b/Graphics/GraphicsEngineD3D12/interface/RenderDeviceFactoryD3D12.h
@@ -48,7 +48,8 @@ public:
                                                Uint32                    NumDeferredContexts) = 0;
 
     virtual void AttachToD3D12Device(void*                      pd3d12NativeDevice, 
-                                     class ICommandQueueD3D12*  pCommandQueue,
+                                     size_t                     CommandQueueCount,
+                                     class ICommandQueueD3D12** ppCommandQueues,
                                      const EngineD3D12Attribs&  EngineAttribs, 
                                      IRenderDevice**            ppDevice, 
                                      IDeviceContext**           ppContexts,
diff --git a/Graphics/GraphicsEngineD3D12/src/BufferD3D12Impl.cpp b/Graphics/GraphicsEngineD3D12/src/BufferD3D12Impl.cpp
index 37c8f1fb..155a8fc6 100644
--- a/Graphics/GraphicsEngineD3D12/src/BufferD3D12Impl.cpp
+++ b/Graphics/GraphicsEngineD3D12/src/BufferD3D12Impl.cpp
@@ -201,12 +201,13 @@ BufferD3D12Impl :: BufferD3D12Impl(IReferenceCounters*          pRefCounters,
             //                  |     N+1, but resource it references    |                                   |
             //                  |     was added to the delete queue      |                                   |
             //                  |     with value N                       |                                   |
-	        pRenderDeviceD3D12->CloseAndExecuteCommandContext(pInitContext, false, nullptr);
+            Uint32 QueueIndex = 0;
+	        pRenderDeviceD3D12->CloseAndExecuteTransientCommandContext(QueueIndex, pInitContext);
 
             // Add reference to the object to the release queue to keep it alive
             // until copy operation is complete. This must be done after
             // submitting command list for execution!
-            pRenderDeviceD3D12->SafeReleaseD3D12Object(UploadBuffer);
+            pRenderDeviceD3D12->SafeReleaseDeviceObject(std::move(UploadBuffer), Uint64{1} << QueueIndex);
         }
 
         if (m_Desc.BindFlags & BIND_UNIFORM_BUFFER)
@@ -292,7 +293,7 @@ BufferD3D12Impl :: ~BufferD3D12Impl()
 {
     // D3D12 object can only be destroyed when it is no longer used by the GPU
     auto *pDeviceD3D12Impl = ValidatedCast<RenderDeviceD3D12Impl>(GetDevice());
-    pDeviceD3D12Impl->SafeReleaseD3D12Object(m_pd3d12Resource);
+    pDeviceD3D12Impl->SafeReleaseDeviceObject(std::move(m_pd3d12Resource), m_Desc.CommandQueueMask);
 }
 
 IMPLEMENT_QUERY_INTERFACE( BufferD3D12Impl, IID_BufferD3D12, TBufferBase )
diff --git a/Graphics/GraphicsEngineD3D12/src/CommandListManager.cpp b/Graphics/GraphicsEngineD3D12/src/CommandListManager.cpp
index f92065ae..98095c1d 100644
--- a/Graphics/GraphicsEngineD3D12/src/CommandListManager.cpp
+++ b/Graphics/GraphicsEngineD3D12/src/CommandListManager.cpp
@@ -64,8 +64,9 @@ void CommandListManager::RequestAllocator(ID3D12CommandAllocator** ppAllocator)
         // later
 		auto& AllocatorPair = m_DiscardedAllocators.front();
         
+        // TODO: Rework
         // Get the last completed fence value
-        auto CompletedFenceValue = m_pDeviceD3D12->GetCompletedFenceValue();
+        auto CompletedFenceValue = m_pDeviceD3D12->GetCompletedFenceValue(0);
         // Note that CompletedFenceValue only grows. So if after we queried
         // the value, the actual value is increased in other thread, this will not
         // be an issue as the only consequence is that potentially available  
diff --git a/Graphics/GraphicsEngineD3D12/src/CommandQueueD3D12Impl.cpp b/Graphics/GraphicsEngineD3D12/src/CommandQueueD3D12Impl.cpp
index ca474933..c3b44870 100644
--- a/Graphics/GraphicsEngineD3D12/src/CommandQueueD3D12Impl.cpp
+++ b/Graphics/GraphicsEngineD3D12/src/CommandQueueD3D12Impl.cpp
@@ -47,29 +47,42 @@ CommandQueueD3D12Impl::~CommandQueueD3D12Impl()
 
 IMPLEMENT_QUERY_INTERFACE( CommandQueueD3D12Impl, IID_CommandQueueD3D12, TBase )
 
-UINT64 CommandQueueD3D12Impl::ExecuteCommandList(ID3D12GraphicsCommandList* commandList)
+Uint64 CommandQueueD3D12Impl::Submit(ID3D12GraphicsCommandList* commandList)
 {
-    ID3D12CommandList *const ppCmdLists[] = {commandList};
-	m_pd3d12CmdQueue->ExecuteCommandLists(1, ppCmdLists);
+    std::lock_guard<std::mutex> Lock(m_QueueMtx);
+
     auto FenceValue = m_NextFenceValue;
-    // Signal the fence
-    m_pd3d12CmdQueue->Signal(m_d3d12Fence, FenceValue);
-    // Increment the value
+    // Increment the value before submitting the list
     Atomics::AtomicIncrement(m_NextFenceValue);
+
+    if (commandList != nullptr)
+    {
+        ID3D12CommandList *const ppCmdLists[] = {commandList};
+	    m_pd3d12CmdQueue->ExecuteCommandLists(1, ppCmdLists);
+    }
+
+    // Signal the fence. This must be done atomically with command list submission.
+    m_pd3d12CmdQueue->Signal(m_d3d12Fence, FenceValue);
+    
     return FenceValue;
 }
 
-void CommandQueueD3D12Impl::IdleGPU()
+Uint64 CommandQueueD3D12Impl::WaitForIdle()
 {
+    std::lock_guard<std::mutex> Lock(m_QueueMtx);
+
     Uint64 LastSignaledFenceValue = m_NextFenceValue;
-    m_pd3d12CmdQueue->Signal(m_d3d12Fence, LastSignaledFenceValue);
     Atomics::AtomicIncrement(m_NextFenceValue);
+
+    m_pd3d12CmdQueue->Signal(m_d3d12Fence, LastSignaledFenceValue);
+    
     if (GetCompletedFenceValue() < LastSignaledFenceValue)
     {
         m_d3d12Fence->SetEventOnCompletion(LastSignaledFenceValue, m_WaitForGPUEventHandle);
         WaitForSingleObject(m_WaitForGPUEventHandle, INFINITE);
         VERIFY(GetCompletedFenceValue() == LastSignaledFenceValue, "Unexpected signaled fence value");
     }
+    return LastSignaledFenceValue;
 }
 
 Uint64 CommandQueueD3D12Impl::GetCompletedFenceValue()
@@ -82,6 +95,7 @@ Uint64 CommandQueueD3D12Impl::GetCompletedFenceValue()
 
 void CommandQueueD3D12Impl::SignalFence(ID3D12Fence* pFence, Uint64 Value)
 {
+    std::lock_guard<std::mutex> Lock(m_QueueMtx);
     m_pd3d12CmdQueue->Signal(pFence, Value);
 }
 
diff --git a/Graphics/GraphicsEngineD3D12/src/DescriptorHeap.cpp b/Graphics/GraphicsEngineD3D12/src/DescriptorHeap.cpp
index 5d6b9a00..d06edf85 100644
--- a/Graphics/GraphicsEngineD3D12/src/DescriptorHeap.cpp
+++ b/Graphics/GraphicsEngineD3D12/src/DescriptorHeap.cpp
@@ -136,7 +136,8 @@ void DescriptorHeapAllocationManager::Free(DescriptorHeapAllocation&& Allocation
     //
     // If basic requirement is met, GetNextFenceValue() will never return a number that is less than the fence value 
     // associated with the last command list that references descriptors from the allocation
-    m_FreeBlockManager.Free(DescriptorOffset, Allocation.GetNumHandles(), m_pDeviceD3D12Impl->GetNextFenceValue());
+    // TODO: Rework
+    m_FreeBlockManager.Free(DescriptorOffset, Allocation.GetNumHandles(), m_pDeviceD3D12Impl->GetNextFenceValue(0));
     
     // Clear the allocation
     Allocation = DescriptorHeapAllocation();
diff --git a/Graphics/GraphicsEngineD3D12/src/PipelineStateD3D12Impl.cpp b/Graphics/GraphicsEngineD3D12/src/PipelineStateD3D12Impl.cpp
index 9e7774ea..964d101e 100644
--- a/Graphics/GraphicsEngineD3D12/src/PipelineStateD3D12Impl.cpp
+++ b/Graphics/GraphicsEngineD3D12/src/PipelineStateD3D12Impl.cpp
@@ -239,7 +239,7 @@ PipelineStateD3D12Impl::~PipelineStateD3D12Impl()
     ShaderResLayoutAllocator.Free(m_pShaderResourceLayouts);
 
     // D3D12 object can only be destroyed when it is no longer used by the GPU
-    m_pDevice->SafeReleaseD3D12Object(m_pd3d12PSO);
+    m_pDevice->SafeReleaseDeviceObject(std::move(m_pd3d12PSO), m_Desc.CommandQueueMask);
 }
 
 IMPLEMENT_QUERY_INTERFACE( PipelineStateD3D12Impl, IID_PipelineStateD3D12, TPipelineStateBase )
diff --git a/Graphics/GraphicsEngineD3D12/src/RenderDeviceD3D12Impl.cpp b/Graphics/GraphicsEngineD3D12/src/RenderDeviceD3D12Impl.cpp
index 9c703876..c341dd48 100644
--- a/Graphics/GraphicsEngineD3D12/src/RenderDeviceD3D12Impl.cpp
+++ b/Graphics/GraphicsEngineD3D12/src/RenderDeviceD3D12Impl.cpp
@@ -41,13 +41,15 @@ RenderDeviceD3D12Impl :: RenderDeviceD3D12Impl(IReferenceCounters*          pRef
                                                IMemoryAllocator&            RawMemAllocator,
                                                const EngineD3D12Attribs&    CreationAttribs,
                                                ID3D12Device*                pd3d12Device,
-                                               ICommandQueueD3D12*          pCmdQueue,
+                                               size_t                       CommandQueueCount,
+                                               ICommandQueueD3D12**         ppCmdQueues, 
                                                Uint32                       NumDeferredContexts) : 
     TRenderDeviceBase
     {
         pRefCounters,
         RawMemAllocator,
-        1,
+        CommandQueueCount,
+        ppCmdQueues,
         NumDeferredContexts,
         sizeof(TextureD3D12Impl),
         sizeof(TextureViewD3D12Impl),
@@ -60,10 +62,7 @@ RenderDeviceD3D12Impl :: RenderDeviceD3D12Impl(IReferenceCounters*          pRef
         sizeof(FenceD3D12Impl)
     },
     m_pd3d12Device  (pd3d12Device),
-    m_pCommandQueue (pCmdQueue),
     m_EngineAttribs (CreationAttribs),
-	m_FrameNumber      (0),
-    m_NextCmdListNumber(0),
     m_CmdListManager(this),
     m_CPUDescriptorHeaps
     {
@@ -84,7 +83,6 @@ RenderDeviceD3D12Impl :: RenderDeviceD3D12Impl(IReferenceCounters*          pRef
 	},
     m_ContextPool(STD_ALLOCATOR_RAW_MEM(ContextPoolElemType, GetRawAllocator(), "Allocator for vector<unique_ptr<CommandContext>>")),
     m_AvailableContexts(STD_ALLOCATOR_RAW_MEM(CommandContext*, GetRawAllocator(), "Allocator for vector<CommandContext*>")),
-    m_ReleaseQueue(GetRawAllocator()),
     m_DynamicMemoryManager(GetRawAllocator(), m_pd3d12Device, CreationAttribs.NumDynamicHeapPagesToReserve, CreationAttribs.DynamicHeapPageSize)
 {
     m_DeviceCaps.DevType = DeviceType::D3D12;
@@ -106,8 +104,11 @@ RenderDeviceD3D12Impl::~RenderDeviceD3D12Impl()
     // release queues
     FinishFrame(true);
 
-    m_DynamicMemoryManager.Destroy(GetCompletedFenceValue());
+    // TODO: Rework
+    m_DynamicMemoryManager.Destroy(GetCompletedFenceValue(0));
+
 	m_ContextPool.clear();
+    DestroyCommandQueues();
 }
 
 void RenderDeviceD3D12Impl::DisposeCommandContext(CommandContext* pCtx)
@@ -116,41 +117,36 @@ void RenderDeviceD3D12Impl::DisposeCommandContext(CommandContext* pCtx)
     m_AvailableContexts.push_back(pCtx);
 }
 
-Uint64 RenderDeviceD3D12Impl::CloseAndExecuteCommandContext(CommandContext* pCtx, bool DiscardStaleObjects, std::vector<std::pair<Uint64, RefCntAutoPtr<IFence> > >* pSignalFences)
+void RenderDeviceD3D12Impl::CloseAndExecuteTransientCommandContext(Uint32 CommandQueueIndex, CommandContext *pCtx)
 {
     CComPtr<ID3D12CommandAllocator> pAllocator;
 	auto *pCmdList = pCtx->Close(&pAllocator);
-
     Uint64 FenceValue = 0;
-    Uint64 CmdListNumber = 0;
-    {
-	    std::lock_guard<std::mutex> LockGuard(m_CmdQueueMutex);
-        auto NextFenceValue = m_pCommandQueue->GetNextFenceValue();
-	    // Submit the command list to the queue
-        FenceValue = m_pCommandQueue->ExecuteCommandList(pCmdList);
-        VERIFY(FenceValue >= NextFenceValue, "Fence value of the executed command list is less than the next fence value previously queried through GetNextFenceValue()");
-        FenceValue = std::max(FenceValue, NextFenceValue);
-        CmdListNumber = m_NextCmdListNumber;
-        Atomics::AtomicIncrement(m_NextCmdListNumber);
-        if (pSignalFences != nullptr)
+    // Execute command list directly through the queue to avoid interference with command list numbers in the queue
+    LockCommandQueue(CommandQueueIndex, 
+        [&](ICommandQueueD3D12* pCmdQueue)
         {
-            for (auto& val_fence : *pSignalFences)
-            {
-                auto* pFenceD3D12Impl = val_fence.second.RawPtr<FenceD3D12Impl>();
-                auto* pd3d12Fence = pFenceD3D12Impl->GetD3D12Fence();
-                m_pCommandQueue->SignalFence(pd3d12Fence, val_fence.first);
-            }
+            FenceValue = pCmdQueue->Submit(pCmdList);
         }
-    }
+    );
+    // DiscardAllocator() is thread-safe
+    // TODO: Rework
+	m_CmdListManager.DiscardAllocator(FenceValue, pAllocator);
 
-    if (DiscardStaleObjects)
     {
-        // The following basic requirement guarantees correctness of resource deallocation:
-        //
-        //        A resource is never released before the last draw command referencing it is invoked on the immediate context
-        //
-        // See http://diligentgraphics.com/diligent-engine/architecture/d3d12/managing-resource-lifetimes/
+	    std::lock_guard<std::mutex> LockGuard(m_ContextAllocationMutex);
+    	m_AvailableContexts.push_back(pCtx);
+    }
+}
+
+Uint64 RenderDeviceD3D12Impl::CloseAndExecuteCommandContext(CommandContext* pCtx, bool DiscardStaleObjects, std::vector<std::pair<Uint64, RefCntAutoPtr<IFence> > >* pSignalFences)
+{
+    CComPtr<ID3D12CommandAllocator> pAllocator;
+	auto *pCmdList = pCtx->Close(&pAllocator);
 
+    Uint32 QueueIndex = 0;
+    Uint64 FenceValue = 0;
+    {
         // Stale objects should only be discarded when submitting cmd list from 
         // the immediate context, otherwise the basic requirement may be violated
         // as in the following scenario
@@ -169,18 +165,24 @@ Uint64 RenderDeviceD3D12Impl::CloseAndExecuteCommandContext(CommandContext* pCtx
         //                  |     N+1, but resource it references    |                                   |
         //                  |     was added to the delete queue      |                                   |
         //                  |     with number N                      |                                   |
-
-        // Move stale objects into a release queue.
-        // Note that objects are moved from stale list to release queue based on the
-        // cmd list number, not the fence value. This makes sure that basic requirement
-        // is met even when the fence value is not incremented while executing 
-        // the command list (as is the case with Unity command queue).
-        m_ReleaseQueue.DiscardStaleResources(CmdListNumber, FenceValue);
+        auto SubmittedCmdBuffInfo = TRenderDeviceBase::SubmitCommandBuffer(QueueIndex, pCmdList, true);
+        FenceValue = SubmittedCmdBuffInfo.FenceValue;
+        if (pSignalFences != nullptr)
+        {
+            for (auto& val_fence : *pSignalFences)
+            {
+                auto* pFenceD3D12Impl = val_fence.second.RawPtr<FenceD3D12Impl>();
+                auto* pd3d12Fence = pFenceD3D12Impl->GetD3D12Fence();
+                m_CommandQueues[QueueIndex].CmdQueue->SignalFence(pd3d12Fence, val_fence.first);
+            }
+        }
     }
 
     // DiscardAllocator() is thread-safe
+    // TODO: Rework
 	m_CmdListManager.DiscardAllocator(FenceValue, pAllocator);
     
+    // TODO: Rework
     pCtx->DiscardDynamicDescriptors(FenceValue);
 
     {
@@ -194,39 +196,10 @@ Uint64 RenderDeviceD3D12Impl::CloseAndExecuteCommandContext(CommandContext* pCtx
 
 void RenderDeviceD3D12Impl::IdleGPU(bool ReleaseStaleObjects) 
 { 
-    Uint64 FenceValue = 0;
-    Uint64 CmdListNumber = 0;
-    {
-        // Lock the command queue to avoid other threads interfering with the GPU
-        std::lock_guard<std::mutex> LockGuard(m_CmdQueueMutex);
-        FenceValue = m_pCommandQueue->GetNextFenceValue();
-        m_pCommandQueue->IdleGPU();
-        // Increment cmd list number while keeping queue locked. 
-        // This guarantees that any D3D12 object released after the lock
-        // is released, will be associated with the incremented cmd list number
-        CmdListNumber = m_NextCmdListNumber;
-        Atomics::AtomicIncrement(m_NextCmdListNumber);
-    }
-    
-    if (ReleaseStaleObjects)
-    {
-        // Do not wait until the end of the frame and force deletion. 
-        // This is necessary to release outstanding references to the
-        // swap chain buffers when it is resized in the middle of the frame.
-        // Since GPU has been idled, it it is safe to do so
-        m_ReleaseQueue.DiscardStaleResources(CmdListNumber, FenceValue);
-        m_ReleaseQueue.Purge(FenceValue);
-    }
-}
-
-Bool RenderDeviceD3D12Impl::IsFenceSignaled(Uint64 FenceValue) 
-{
-    return FenceValue <= GetCompletedFenceValue();
-}
-
-Uint64 RenderDeviceD3D12Impl::GetCompletedFenceValue()
-{
-    return m_pCommandQueue->GetCompletedFenceValue();
+    // Do not wait until the end of the frame and force deletion. 
+    // It is necessary to release outstanding references to the
+    // swap chain buffers when it is resized in the middle of the frame.
+    IdleCommandQueues(ReleaseStaleObjects);
 }
 
 void RenderDeviceD3D12Impl::FinishFrame(bool ReleaseAllResources)
@@ -250,24 +223,16 @@ void RenderDeviceD3D12Impl::FinishFrame(bool ReleaseAllResources)
         }
     }
     
-    auto CompletedFenceValue = ReleaseAllResources ? std::numeric_limits<Uint64>::max() : GetCompletedFenceValue();
-
-    // We must use NextFenceValue here, NOT current value, because the 
-    // fence value may or may not have been incremented when the last 
-    // command list was submitted for execution (Unity only
-    // increments fence value once per frame)
-    Uint64 NextFenceValue = 0;
-    Uint64 CmdListNumber = 0;
-    {
-        // Lock the command queue to avoid other threads interfering with the GPU
-        std::lock_guard<std::mutex> LockGuard(m_CmdQueueMutex);
-        NextFenceValue = m_pCommandQueue->GetNextFenceValue();
-        // Increment cmd list number while keeping queue locked. 
-        // This guarantees that any D3D12 object released after the lock
-        // is released, will be associated with the incremented cmd list number
-        CmdListNumber = m_NextCmdListNumber;
-        Atomics::AtomicIncrement(m_NextCmdListNumber);
-    }
+    // TODO: rework
+    // Submit empty command list to set a fence on the GPU and increment command list number.
+    // Discard all remaining objects which is important to do if there were 
+    // no command lists submitted during the frame
+    ID3D12GraphicsCommandList* pNullCmdList = nullptr;
+    TRenderDeviceBase::SubmitCommandBuffer(0, pNullCmdList, true);
+
+    auto CompletedFenceValue = ReleaseAllResources ? std::numeric_limits<Uint64>::max() : GetCompletedFenceValue(0);
+   
+    PurgeReleaseQueues();
 
     // Dynamic memory is used to update resource contents as well as to allocate
     // space for dynamic resources.
@@ -285,13 +250,6 @@ void RenderDeviceD3D12Impl::FinishFrame(bool ReleaseAllResources)
     {
         m_GPUDescriptorHeaps[GPUHeap].ReleaseStaleAllocations(CompletedFenceValue);
     }
-
-    // Discard all remaining objects. This is important to do if there were 
-    // no command lists submitted during the frame
-    m_ReleaseQueue.DiscardStaleResources(CmdListNumber, NextFenceValue);
-    m_ReleaseQueue.Purge(CompletedFenceValue);
-
-    Atomics::AtomicIncrement(m_FrameNumber);
 }
 
 
@@ -321,14 +279,6 @@ CommandContext* RenderDeviceD3D12Impl::AllocateCommandContext(const Char* ID)
 	return ret;
 }
 
-void RenderDeviceD3D12Impl::SafeReleaseD3D12Object(ID3D12Object* pObj)
-{
-    // When D3D12 object is released, it is first moved into the
-    // stale objects list. The list is moved into a release queue
-    // when the next command list is executed. 
-    m_ReleaseQueue.SafeReleaseResource(CComPtr<ID3D12Object>(pObj), m_NextCmdListNumber);
-}
-
 bool CreateTestResource(ID3D12Device* pDevice, const D3D12_RESOURCE_DESC& ResDesc)
 {
     // Set the texture pointer address to nullptr to validate input parameters
diff --git a/Graphics/GraphicsEngineD3D12/src/RenderDeviceFactoryD3D12.cpp b/Graphics/GraphicsEngineD3D12/src/RenderDeviceFactoryD3D12.cpp
index 90d15a55..4dc46ff0 100644
--- a/Graphics/GraphicsEngineD3D12/src/RenderDeviceFactoryD3D12.cpp
+++ b/Graphics/GraphicsEngineD3D12/src/RenderDeviceFactoryD3D12.cpp
@@ -25,6 +25,7 @@
 /// Routines that initialize D3D12-based engine implementation
 
 #include "pch.h"
+#include <array>
 #include "RenderDeviceFactoryD3D12.h"
 #include "RenderDeviceD3D12Impl.h"
 #include "DeviceContextD3D12Impl.h"
@@ -56,7 +57,8 @@ public:
                                        Uint32                    NumDeferredContexts)override final;
 
     void AttachToD3D12Device(void*                     pd3d12NativeDevice, 
-                             ICommandQueueD3D12*       pCommandQueue,
+                             size_t                    CommandQueueCount,
+                             ICommandQueueD3D12**      ppCommandQueues,
                              const EngineD3D12Attribs& EngineAttribs, 
                              IRenderDevice**           ppDevice, 
                              IDeviceContext**          ppContexts,
@@ -260,14 +262,16 @@ void EngineFactoryD3D12Impl::CreateDeviceAndContextsD3D12( const EngineD3D12Attr
         return;
     }
         
-    AttachToD3D12Device(d3d12Device, pCmdQueueD3D12, CreationAttribs, ppDevice, ppContexts, NumDeferredContexts);
+    std::array<ICommandQueueD3D12*, 1> CmdQueues = {pCmdQueueD3D12};
+    AttachToD3D12Device(d3d12Device, CmdQueues.size(), CmdQueues.data(), CreationAttribs, ppDevice, ppContexts, NumDeferredContexts);
 }
 
 
 /// Attaches to existing D3D12 device
 
 /// \param [in] pd3d12NativeDevice - pointer to native D3D12 device
-/// \param [in] pCommandQueue - pointer to the implementation of command queue
+/// \param [in] CommandQueueCount - Number of command queues
+/// \param [in] ppCommandQueues - pointer to the array of command queues
 /// \param [in] EngineAttribs - Engine creation attributes.
 /// \param [out] ppDevice - Address of the memory location where pointer to 
 ///                         the created device will be written
@@ -280,7 +284,8 @@ void EngineFactoryD3D12Impl::CreateDeviceAndContextsD3D12( const EngineD3D12Attr
 ///                                   contexts are written to ppContexts array starting 
 ///                                   at position 1
 void EngineFactoryD3D12Impl::AttachToD3D12Device(void*                     pd3d12NativeDevice, 
-                                                 ICommandQueueD3D12*       pCommandQueue,
+                                                 size_t                    CommandQueueCount,
+                                                 ICommandQueueD3D12**      ppCommandQueues,
                                                  const EngineD3D12Attribs& EngineAttribs, 
                                                  IRenderDevice**           ppDevice, 
                                                  IDeviceContext**          ppContexts,
@@ -289,8 +294,8 @@ void EngineFactoryD3D12Impl::AttachToD3D12Device(void*                     pd3d1
     if (EngineAttribs.DebugMessageCallback != nullptr)
         SetDebugMessageCallback(EngineAttribs.DebugMessageCallback);
 
-    VERIFY( pd3d12NativeDevice && pCommandQueue && ppDevice && ppContexts, "Null pointer provided" );
-    if( !pd3d12NativeDevice || !pCommandQueue || !ppDevice || !ppContexts )
+    VERIFY( pd3d12NativeDevice && ppCommandQueues && ppDevice && ppContexts, "Null pointer provided" );
+    if( !pd3d12NativeDevice || !ppCommandQueues || !ppDevice || !ppContexts )
         return;
 
     *ppDevice = nullptr;
@@ -301,7 +306,7 @@ void EngineFactoryD3D12Impl::AttachToD3D12Device(void*                     pd3d1
         SetRawAllocator(EngineAttribs.pRawMemAllocator);
         auto &RawMemAllocator = GetRawAllocator();
         auto d3d12Device = reinterpret_cast<ID3D12Device*>(pd3d12NativeDevice);
-        RenderDeviceD3D12Impl *pRenderDeviceD3D12( NEW_RC_OBJ(RawMemAllocator, "RenderDeviceD3D12Impl instance", RenderDeviceD3D12Impl)(RawMemAllocator, EngineAttribs, d3d12Device, pCommandQueue, NumDeferredContexts ) );
+        RenderDeviceD3D12Impl *pRenderDeviceD3D12( NEW_RC_OBJ(RawMemAllocator, "RenderDeviceD3D12Impl instance", RenderDeviceD3D12Impl)(RawMemAllocator, EngineAttribs, d3d12Device, CommandQueueCount, ppCommandQueues, NumDeferredContexts ) );
         pRenderDeviceD3D12->QueryInterface(IID_RenderDevice, reinterpret_cast<IObject**>(ppDevice) );
 
         RefCntAutoPtr<DeviceContextD3D12Impl> pImmediateCtxD3D12( NEW_RC_OBJ(RawMemAllocator, "DeviceContextD3D12Impl instance", DeviceContextD3D12Impl)(pRenderDeviceD3D12, false, EngineAttribs, 0) );
diff --git a/Graphics/GraphicsEngineD3D12/src/SwapChainD3D12Impl.cpp b/Graphics/GraphicsEngineD3D12/src/SwapChainD3D12Impl.cpp
index deab12e8..d01d4ab5 100644
--- a/Graphics/GraphicsEngineD3D12/src/SwapChainD3D12Impl.cpp
+++ b/Graphics/GraphicsEngineD3D12/src/SwapChainD3D12Impl.cpp
@@ -41,8 +41,12 @@ SwapChainD3D12Impl::SwapChainD3D12Impl(IReferenceCounters*       pRefCounters,
     TSwapChainBase(pRefCounters, pRenderDeviceD3D12, pDeviceContextD3D12, SCDesc, FSDesc, pNativeWndHandle),
     m_pBackBufferRTV(STD_ALLOCATOR_RAW_MEM(RefCntAutoPtr<ITextureView>, GetRawAllocator(), "Allocator for vector<RefCntAutoPtr<ITextureView>>"))
 {
-    auto *pd3d12CmdQueue = pRenderDeviceD3D12->GetCmdQueue()->GetD3D12CommandQueue();
-    CreateDXGISwapChain(pd3d12CmdQueue);
+    pRenderDeviceD3D12->LockCommandQueue(0, 
+        [this](ICommandQueueD3D12 *pCmdQueue)
+        {
+            CreateDXGISwapChain(pCmdQueue->GetD3D12CommandQueue());
+        }
+    );
     InitBuffersAndViews();
 }
 
@@ -173,8 +177,12 @@ void SwapChainD3D12Impl::UpdateSwapChain(bool CreateNew)
             if(CreateNew)
             {
                 m_pSwapChain.Release();
-                auto *pd3d12CmdQueue = m_pRenderDevice.RawPtr<RenderDeviceD3D12Impl>()->GetCmdQueue()->GetD3D12CommandQueue();
-                CreateDXGISwapChain(pd3d12CmdQueue);
+                m_pRenderDevice.RawPtr<RenderDeviceD3D12Impl>()->LockCommandQueue(0, 
+                    [this](ICommandQueueD3D12 *pCmdQueue)
+                    {
+                        CreateDXGISwapChain(pCmdQueue->GetD3D12CommandQueue());
+                    }
+                );
             }
             else
             {
diff --git a/Graphics/GraphicsEngineD3D12/src/TextureD3D12Impl.cpp b/Graphics/GraphicsEngineD3D12/src/TextureD3D12Impl.cpp
index 96bdc406..e76da5dd 100644
--- a/Graphics/GraphicsEngineD3D12/src/TextureD3D12Impl.cpp
+++ b/Graphics/GraphicsEngineD3D12/src/TextureD3D12Impl.cpp
@@ -218,7 +218,8 @@ TextureD3D12Impl :: TextureD3D12Impl(IReferenceCounters*        pRefCounters,
         //                  |     N+1, but resource it references    |                                   |
         //                  |     was added to the delete queue      |                                   |
         //                  |     with value N                       |                                   |
-	    pRenderDeviceD3D12->CloseAndExecuteCommandContext(pInitContext, false, nullptr);
+        Uint32 QueueIndex = 0;
+	    pRenderDeviceD3D12->CloseAndExecuteTransientCommandContext(QueueIndex, pInitContext);
 
         // We MUST NOT call TransitionResource() from here, because
         // it will call AddRef() and potentially Release(), while 
@@ -226,7 +227,7 @@ TextureD3D12Impl :: TextureD3D12Impl(IReferenceCounters*        pRefCounters,
         // Add reference to the object to the release queue to keep it alive
         // until copy operation is complete.  This must be done after
         // submitting command list for execution!
-        pRenderDeviceD3D12->SafeReleaseD3D12Object(UploadBuffer);
+        pRenderDeviceD3D12->SafeReleaseDeviceObject(std::move(UploadBuffer), Uint64{1} << QueueIndex);
     }
 
     if(m_Desc.MiscFlags & MISC_TEXTURE_FLAG_GENERATE_MIPS)
@@ -398,7 +399,7 @@ TextureD3D12Impl :: ~TextureD3D12Impl()
 {
     // D3D12 object can only be destroyed when it is no longer used by the GPU
     auto *pDeviceD3D12Impl = ValidatedCast<RenderDeviceD3D12Impl>(GetDevice());
-    pDeviceD3D12Impl->SafeReleaseD3D12Object(m_pd3d12Resource);
+    pDeviceD3D12Impl->SafeReleaseDeviceObject(std::move(m_pd3d12Resource), m_Desc.CommandQueueMask);
 }
 
 void TextureD3D12Impl::UpdateData( IDeviceContext*          pContext,