Reworked upload heap in Vulkan to suballocate pages from the global resource manager

8 files changed, 301 insertions, 226 deletions

diff --git a/Graphics/GraphicsEngineVulkan/CMakeLists.txt b/Graphics/GraphicsEngineVulkan/CMakeLists.txt
index 45bc9595..e920fcae 100644
--- a/Graphics/GraphicsEngineVulkan/CMakeLists.txt
+++ b/Graphics/GraphicsEngineVulkan/CMakeLists.txt
@@ -30,6 +30,7 @@ set(INCLUDE
     include/TextureViewVkImpl.h
     include/VulkanErrors.h
     include/VulkanTypeConversions.h
+    include/VulkanUploadHeap.h
 )
 
 set(VULKAN_UTILS_INCLUDE 
@@ -43,7 +44,6 @@ set(VULKAN_UTILS_INCLUDE
     include/VulkanUtilities/VulkanMemoryManager.h
     include/VulkanUtilities/VulkanObjectWrappers.h
     include/VulkanUtilities/VulkanPhysicalDevice.h
-    include/VulkanUtilities/VulkanUploadHeap.h
 )
 
 
@@ -91,6 +91,7 @@ set(SRC
     src/TextureVkImpl.cpp
     src/TextureViewVkImpl.cpp
     src/VulkanTypeConversions.cpp
+    src/VulkanUploadHeap.cpp
 )
 
 set(VULKAN_UTILS_SRC
@@ -103,7 +104,6 @@ set(VULKAN_UTILS_SRC
     src/VulkanUtilities/VulkanLogicalDevice.cpp
     src/VulkanUtilities/VulkanMemoryManager.cpp
     src/VulkanUtilities/VulkanPhysicalDevice.cpp
-    src/VulkanUtilities/VulkanUploadHeap.cpp
 )
 
 set(GENERATE_MIPS_SHADER
diff --git a/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h b/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h
index e7483c22..94c0a3c8 100644
--- a/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h
+++ b/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h
@@ -31,7 +31,7 @@
 #include "DeviceContextBase.h"
 #include "VulkanUtilities/VulkanCommandBufferPool.h"
 #include "VulkanUtilities/VulkanCommandBuffer.h"
-#include "VulkanUtilities/VulkanUploadHeap.h"
+#include "VulkanUploadHeap.h"
 #include "VulkanDynamicHeap.h"
 #include "ResourceReleaseQueue.h"
 #include "DescriptorPoolManager.h"
@@ -213,10 +213,9 @@ private:
     // List of fences to signal next time the command context is flushed
     std::vector<std::pair<Uint64, RefCntAutoPtr<IFence> > > m_PendingFences;
 
-    std::unordered_map<BufferVkImpl*, VulkanUtilities::VulkanUploadAllocation> m_UploadAllocations;
-    ResourceReleaseQueue<DynamicStaleResourceWrapper> m_ReleaseQueue;
+    std::unordered_map<BufferVkImpl*, VulkanUploadAllocation> m_UploadAllocations;
 
-    VulkanUtilities::VulkanUploadHeap m_UploadHeap;
+    VulkanUploadHeap m_UploadHeap;
     DescriptorPoolManager m_DynamicDescriptorPool;
 
     // Number of the command buffer currently being recorded by the context and that will
diff --git a/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h b/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h
index 9fbe7c32..5dc4514d 100644
--- a/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h
+++ b/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h
@@ -39,7 +39,7 @@
 #include "VulkanUtilities/VulkanLogicalDevice.h"
 #include "VulkanUtilities/VulkanObjectWrappers.h"
 #include "VulkanUtilities/VulkanMemoryManager.h"
-#include "VulkanUtilities/VulkanUploadHeap.h"
+#include "VulkanUploadHeap.h"
 #include "FramebufferCache.h"
 #include "RenderPassCache.h"
 #include "CommandPoolManager.h"
@@ -113,6 +113,8 @@ public:
     {
         m_ReleaseQueue.SafeReleaseResource(std::move(Object), m_NextCmdBuffNumber);
     }
+
+    ResourceReleaseQueue<DynamicStaleResourceWrapper>& GetReleaseQueue(){return m_ReleaseQueue;}
     
     void FinishFrame(bool ReleaseAllResources);
     virtual void FinishFrame()override final { FinishFrame(false); }
@@ -132,8 +134,9 @@ public:
     {
         return m_MemoryMgr.Allocate(MemReqs, MemoryProperties);
     }
+    VulkanUtilities::VulkanMemoryManager& GetGlobalMemoryManager() { return m_MemoryMgr; }
 
-    VulkanDynamicMemoryManager& GetDynamicMemoryManager(){return m_DynamicMemoryManager;}
+    VulkanDynamicMemoryManager& GetDynamicMemoryManager() { return m_DynamicMemoryManager; }
      
 private:
     virtual void TestTextureFormat( TEXTURE_FORMAT TexFormat )override final;
diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanUploadHeap.h b/Graphics/GraphicsEngineVulkan/include/VulkanUploadHeap.h
new file mode 100644
index 00000000..ed9c6707
--- /dev/null
+++ b/Graphics/GraphicsEngineVulkan/include/VulkanUploadHeap.h
@@ -0,0 +1,118 @@
+/*     Copyright 2015-2018 Egor Yusov
+ *  
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF ANY PROPRIETARY RIGHTS.
+ *
+ *  In no event and under no legal theory, whether in tort (including negligence), 
+ *  contract, or otherwise, unless required by applicable law (such as deliberate 
+ *  and grossly negligent acts) or agreed to in writing, shall any Contributor be
+ *  liable for any damages, including any direct, indirect, special, incidental, 
+ *  or consequential damages of any character arising as a result of this License or 
+ *  out of the use or inability to use the software (including but not limited to damages 
+ *  for loss of goodwill, work stoppage, computer failure or malfunction, or any and 
+ *  all other commercial damages or losses), even if such Contributor has been advised 
+ *  of the possibility of such damages.
+ */
+
+#pragma once
+
+#include <unordered_map>
+#include "VulkanUtilities/VulkanMemoryManager.h"
+
+namespace Diligent
+{
+
+class RenderDeviceVkImpl;
+
+struct VulkanUploadAllocation
+{
+    VulkanUploadAllocation(){}
+    VulkanUploadAllocation(void* _CPUAddress, VkDeviceSize _Size, VkDeviceSize _Offset, VkBuffer _vkBuffer) :
+        CPUAddress(_CPUAddress),
+        Size      (_Size),
+        Offset    (_Offset),
+        vkBuffer  (_vkBuffer)
+    {}
+    VulkanUploadAllocation             (const VulkanUploadAllocation&) = delete;
+    VulkanUploadAllocation& operator = (const VulkanUploadAllocation&) = delete;
+    VulkanUploadAllocation             (VulkanUploadAllocation&&) = default;
+    VulkanUploadAllocation& operator = (VulkanUploadAllocation&&) = default;
+
+    VkBuffer     vkBuffer   = VK_NULL_HANDLE;	    // Vulkan buffer associated with this memory.
+    void*        CPUAddress = nullptr;
+    VkDeviceSize Size       = 0;
+    VkDeviceSize Offset     = 0;
+};
+
+class VulkanUploadHeap
+{
+public:
+    VulkanUploadHeap(RenderDeviceVkImpl& RenderDevice,
+                     std::string         HeapName,
+                     VkDeviceSize        PageSize);
+    
+    VulkanUploadHeap            (VulkanUploadHeap&&)      = delete;
+    VulkanUploadHeap            (const VulkanUploadHeap&) = delete;
+    VulkanUploadHeap& operator= (VulkanUploadHeap&)       = delete;
+    VulkanUploadHeap& operator= (VulkanUploadHeap&& rhs)  = delete;
+
+    ~VulkanUploadHeap();
+
+    VulkanUploadAllocation Allocate(size_t SizeInBytes);
+    void DiscardAllocations(uint64_t FenceValue);
+
+    size_t GetStaleAllocationsCount()const
+    {
+        return m_Pages.size();
+    }
+
+private:
+    RenderDeviceVkImpl&                   m_RenderDevice;
+    std::string                           m_HeapName;
+    struct UploadPageInfo
+    {
+        VulkanUtilities::VulkanMemoryAllocation MemAllocation;
+        VulkanUtilities::BufferWrapper          Buffer;
+        Uint8*                                  CPUAddress = nullptr;
+    };
+    std::vector<UploadPageInfo> m_Pages;
+
+    struct CurrPageInfo
+    {
+        VkBuffer vkBuffer       = VK_NULL_HANDLE;
+        Uint8*   CPUAddress     = nullptr;
+        size_t   CurrOffset     = 0;
+        size_t   AvailableSize  = 0;
+        void Reset(UploadPageInfo& NewPage)
+        {
+            vkBuffer       = NewPage.Buffer;
+            CPUAddress     = NewPage.CPUAddress;
+            CurrOffset     = 0;
+            AvailableSize  = NewPage.MemAllocation.Size;
+        }
+        void Advance(size_t SizeInBytes)
+        {
+            CPUAddress    += SizeInBytes;
+            CurrOffset    += SizeInBytes;
+            AvailableSize -= SizeInBytes;
+        }
+    }m_CurrPage;
+
+    size_t   m_CurrFrameSize   = 0;
+    size_t   m_PeakFrameSize   = 0;
+    size_t   m_CurrAllocatedSize   = 0;
+    size_t   m_PeakAllocatedSize   = 0;
+
+    const VkDeviceSize m_PageSize;
+
+    UploadPageInfo CreateNewPage(VkDeviceSize SizeInBytes);
+};
+
+}
diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanUploadHeap.h b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanUploadHeap.h
deleted file mode 100644
index 1302f795..00000000
--- a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanUploadHeap.h
+++ /dev/null
@@ -1,84 +0,0 @@
-/*     Copyright 2015-2018 Egor Yusov
- *  
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF ANY PROPRIETARY RIGHTS.
- *
- *  In no event and under no legal theory, whether in tort (including negligence), 
- *  contract, or otherwise, unless required by applicable law (such as deliberate 
- *  and grossly negligent acts) or agreed to in writing, shall any Contributor be
- *  liable for any damages, including any direct, indirect, special, incidental, 
- *  or consequential damages of any character arising as a result of this License or 
- *  out of the use or inability to use the software (including but not limited to damages 
- *  for loss of goodwill, work stoppage, computer failure or malfunction, or any and 
- *  all other commercial damages or losses), even if such Contributor has been advised 
- *  of the possibility of such damages.
- */
-
-#pragma once
-
-#include <unordered_map>
-#include "VulkanUtilities/VulkanMemoryManager.h"
-
-namespace VulkanUtilities
-{
-
-struct VulkanUploadAllocation
-{
-    VulkanUploadAllocation(){}
-    VulkanUploadAllocation(VulkanMemoryAllocation&& _MemAllocation, VkBuffer _vkBuffer) :
-        MemAllocation(std::move(_MemAllocation)),
-        vkBuffer     (_vkBuffer)
-    {}
-    VulkanUploadAllocation             (const VulkanUploadAllocation&) = delete;
-    VulkanUploadAllocation& operator = (const VulkanUploadAllocation&) = delete;
-    VulkanUploadAllocation             (VulkanUploadAllocation&&) = default;
-    VulkanUploadAllocation& operator = (VulkanUploadAllocation&&) = default;
-
-    VulkanMemoryAllocation MemAllocation;
-    VkBuffer vkBuffer = VK_NULL_HANDLE;	    // Vulkan buffer associated with this memory.
-};
-
-class VulkanUploadHeap : public VulkanMemoryManager
-{
-public:
-    VulkanUploadHeap(std::string                  MgrName,
-                     const VulkanLogicalDevice&   LogicalDevice, 
-                     const VulkanPhysicalDevice&  PhysicalDevice, 
-                     Diligent::IMemoryAllocator&  Allocator, 
-                     VkDeviceSize                 HostVisiblePageSize,
-                     VkDeviceSize                 HostVisibleReserveSize);
-    
-    VulkanUploadHeap(VulkanUploadHeap&& rhs)noexcept :
-        VulkanMemoryManager           (std::move(rhs)),
-        m_StagingBufferMemoryTypeIndex(rhs.m_StagingBufferMemoryTypeIndex),
-        m_Buffers                     (std::move(rhs.m_Buffers))
-    {
-    }
-    VulkanUploadHeap            (const VulkanUploadHeap&) = delete;
-    VulkanUploadHeap& operator= (VulkanUploadHeap&)       = delete;
-    VulkanUploadHeap& operator= (VulkanUploadHeap&& rhs)  = delete;
-
-    ~VulkanUploadHeap();
-
-    VulkanUploadAllocation Allocate(size_t SizeInBytes);
-
-private:
-    virtual void OnNewPageCreated(VulkanMemoryPage& NewPage);
-    virtual void OnPageDestroy(VulkanMemoryPage& Page);
-    VkBufferCreateInfo GetStagingBufferCI()const;
-    
-    uint32_t m_StagingBufferMemoryTypeIndex = 0;
-    std::mutex m_BuffersMtx;
-    std::unordered_map<VulkanMemoryPage*, VulkanUtilities::BufferWrapper> m_Buffers;
-
-    // If adding new member, do not forget to update move ctor
-};
-
-}
diff --git a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
index c8b95867..f469e722 100644
--- a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
@@ -76,16 +76,12 @@ namespace Diligent
             VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, 
             bIsDeferred
         },
-        m_ReleaseQueue{GetRawAllocator()},
         // Upload heap must always be thread-safe as Finish() may be called from another thread
         m_UploadHeap
         {
+            *pDeviceVkImpl,
             GetUploadHeapName(bIsDeferred, ContextId),
-            pDeviceVkImpl->GetLogicalDevice(),
-            pDeviceVkImpl->GetPhysicalDevice(),
-            GetRawAllocator(),
-            bIsDeferred ? Attribs.DeferredCtxUploadHeapPageSize    : Attribs.ImmediateCtxUploadHeapPageSize,
-            bIsDeferred ? Attribs.DeferredCtxUploadHeapReserveSize : Attribs.ImmediateCtxUploadHeapReserveSize
+            Attribs.UploadHeapPageSize
         },
         // Descriptor pools must always be thread-safe as for a deferred context, Finish() may be called from another thread
         m_DynamicDescriptorPool
@@ -112,7 +108,7 @@ namespace Diligent
         {
             pDeviceVkImpl->GetDynamicMemoryManager(),
             GetDynamicHeapName(bIsDeferred, ContextId),
-            bIsDeferred ? Attribs.DeferredCtxDynamicHeapPageSize : Attribs.ImmediateCtxDynamicHeapPageSize
+            Attribs.DynamicHeapPageSize
         },
         m_GenerateMipsHelper(std::move(GenerateMipsHelper))
     {
@@ -144,11 +140,10 @@ namespace Diligent
         // There must be no resources in the stale resource list. For immediate context, all stale resources must have been
         // moved to the release queue by Flush(). For deferred contexts, this should have happened in the last FinishCommandList()
         // call.
-        VERIFY(m_ReleaseQueue.GetStaleResourceCount() == 0, "All stale resources must have been discarded at this point");
+        VERIFY(m_UploadHeap.GetStaleAllocationsCount() == 0, "All stale allocations must have been discarded at this point");
         VERIFY(m_DynamicDescriptorPool.GetStaleAllocationCount() == 0, "All stale dynamic descriptor set allocations must have been discarded at this point");
         ReleaseStaleContextResources(m_NextCmdBuffNumber, m_LastSubmittedFenceValue, pDeviceVkImpl->GetCompletedFenceValue());
         // Since we idled the GPU, all stale resources must have been destroyed now
-        VERIFY(m_ReleaseQueue.GetPendingReleaseResourceCount() == 0, "All stale resources must have been destroyed at this point");
         VERIFY(m_DynamicDescriptorPool.GetPendingReleaseAllocationCount() == 0, "All stale descriptor set allocations must have been destroyed at this point");
 
         auto VkCmdPool = m_CmdPool.Release();
@@ -767,8 +762,7 @@ namespace Diligent
                 "There are outstanding commands in the deferred device context when finishing the frame. This is an error and may cause unpredicted behaviour. Close all deferred contexts and execute them before finishing the frame" :
                 "There are outstanding commands in the immediate device context when finishing the frame. This is an error and may cause unpredicted behaviour. Call Flush() to submit all commands for execution before finishing the frame");
 
-        m_ReleaseQueue.Purge(CompletedFenceValue);
-        m_UploadHeap.ShrinkMemory();
+        m_UploadHeap.DiscardAllocations(m_LastSubmittedFenceValue);
         m_DynamicDescriptorPool.ReleaseStaleAllocations(CompletedFenceValue);
         m_DynamicHeap.FinishFrame(m_LastSubmittedFenceValue);
         Atomics::AtomicIncrement(m_ContextFrameNumber);
@@ -776,9 +770,6 @@ namespace Diligent
 
     void DeviceContextVkImpl::ReleaseStaleContextResources(Uint64 SubmittedCmdBufferNumber, Uint64 SubmittedFenceValue, Uint64 CompletedFenceValue)
     {
-        m_ReleaseQueue.DiscardStaleResources(SubmittedCmdBufferNumber, SubmittedFenceValue);
-        m_ReleaseQueue.Purge(CompletedFenceValue);
-        m_UploadHeap.ShrinkMemory();
         m_DynamicDescriptorPool.DisposeAllocations(SubmittedFenceValue);
         m_DynamicDescriptorPool.ReleaseStaleAllocations(CompletedFenceValue);
     }
@@ -1099,13 +1090,10 @@ namespace Diligent
 
         VERIFY_EXPR( static_cast<size_t>(NumBytes) == NumBytes );
         auto TmpSpace = m_UploadHeap.Allocate(static_cast<size_t>(NumBytes));
-        auto CPUAddress = TmpSpace.MemAllocation.Page->GetCPUMemory();
-	    memcpy(reinterpret_cast<Uint8*>(CPUAddress) + TmpSpace.MemAllocation.UnalignedOffset, pData, static_cast<size_t>(NumBytes));
-        UpdateBufferRegion(pBuffVk, DstOffset, NumBytes, TmpSpace.vkBuffer, TmpSpace.MemAllocation.UnalignedOffset);
-        // The allocation will stay in the queue until the command buffer from this context is submitted
-        // to the queue. We cannot use the device's release queue as other contexts may interfer with
-        // the release order
-        m_ReleaseQueue.SafeReleaseResource(std::move(TmpSpace), m_NextCmdBuffNumber);
+	    memcpy(TmpSpace.CPUAddress, pData, static_cast<size_t>(NumBytes));
+        UpdateBufferRegion(pBuffVk, DstOffset, NumBytes, TmpSpace.vkBuffer, TmpSpace.Offset);
+        // The allocation will stay in the upload heap until the end of the frame at which point all upload
+        // pages will be discarded
     }
 
     void DeviceContextVkImpl::CopyBufferRegion(BufferVkImpl *pSrcBuffVk, BufferVkImpl *pDstBuffVk, Uint64 SrcOffset, Uint64 DstOffset, Uint64 NumBytes)
@@ -1297,8 +1285,6 @@ namespace Diligent
         // We can now release all temporary resources in the deferred context associated with the submitted command list
         auto CompletedFenceValue = pDeviceVkImpl->GetCompletedFenceValue();
         pDeferredCtxVkImpl->ReleaseStaleContextResources(DeferredCtxCmdBuffNumber, SubmittedFenceValue, CompletedFenceValue);
-        
-        m_ReleaseQueue.Purge(CompletedFenceValue);
     }
 
     void DeviceContextVkImpl::SignalFence(IFence* pFence, Uint64 Value)
@@ -1379,7 +1365,7 @@ namespace Diligent
     {
         VERIFY(m_UploadAllocations.find(pBuffer) == m_UploadAllocations.end(), "Upload space has already been allocated for this buffer");
         auto UploadAllocation = m_UploadHeap.Allocate(NumBytes);
-        auto *CPUAddress = reinterpret_cast<Uint8*>(UploadAllocation.MemAllocation.Page->GetCPUMemory()) + UploadAllocation.MemAllocation.UnalignedOffset;
+        auto *CPUAddress = UploadAllocation.CPUAddress;
         m_UploadAllocations.emplace(pBuffer, std::move(UploadAllocation));
         return CPUAddress;
     }
@@ -1389,12 +1375,10 @@ namespace Diligent
         auto it = m_UploadAllocations.find(pBuffer);
         if (it != m_UploadAllocations.end())
         {
-            VERIFY_EXPR(pBuffer->GetDesc().uiSizeInBytes <= it->second.MemAllocation.Size);
-            UpdateBufferRegion(pBuffer, 0, pBuffer->GetDesc().uiSizeInBytes, it->second.vkBuffer, it->second.MemAllocation.UnalignedOffset);
-            // The allocation will stay in the queue until the command buffer from this context is submitted
-            // to the queue. We cannot use the device's release queue as other contexts may interfer with
-            // the release order
-            m_ReleaseQueue.SafeReleaseResource(std::move(it->second), m_NextCmdBuffNumber);
+            VERIFY_EXPR(pBuffer->GetDesc().uiSizeInBytes <= it->second.Size);
+            UpdateBufferRegion(pBuffer, 0, pBuffer->GetDesc().uiSizeInBytes, it->second.vkBuffer, it->second.Offset);
+            // The allocation will stay in the upload heap until the end of the frame at which point all upload
+            // pages will be discarded
             m_UploadAllocations.erase(it);
         }
         else
diff --git a/Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp b/Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp
new file mode 100644
index 00000000..685a3d13
--- /dev/null
+++ b/Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp
@@ -0,0 +1,159 @@
+/*     Copyright 2015-2018 Egor Yusov
+ *  
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF ANY PROPRIETARY RIGHTS.
+ *
+ *  In no event and under no legal theory, whether in tort (including negligence), 
+ *  contract, or otherwise, unless required by applicable law (such as deliberate 
+ *  and grossly negligent acts) or agreed to in writing, shall any Contributor be
+ *  liable for any damages, including any direct, indirect, special, incidental, 
+ *  or consequential damages of any character arising as a result of this License or 
+ *  out of the use or inability to use the software (including but not limited to damages 
+ *  for loss of goodwill, work stoppage, computer failure or malfunction, or any and 
+ *  all other commercial damages or losses), even if such Contributor has been advised 
+ *  of the possibility of such damages.
+ */
+
+#include "pch.h"
+#include "VulkanUploadHeap.h"
+#include "RenderDeviceVkImpl.h"
+
+namespace Diligent
+{
+
+VulkanUploadHeap::VulkanUploadHeap(RenderDeviceVkImpl& RenderDevice,
+                                   std::string         HeapName,
+                                   VkDeviceSize        PageSize) :
+    m_RenderDevice (RenderDevice),
+    m_HeapName     (std::move(HeapName)),
+    m_PageSize     (PageSize)
+{
+}
+
+VulkanUploadHeap::~VulkanUploadHeap()
+{
+    DEV_CHECK_ERR(m_Pages.empty(), "Upload heap '", m_HeapName, "' not all pages are released");
+    LOG_INFO_MESSAGE(m_HeapName, " peak used/peak allocated frame size: ", FormatMemorySize(m_PeakFrameSize, 2, m_PeakAllocatedSize), '/', FormatMemorySize(m_PeakAllocatedSize, 2));
+}
+
+VulkanUploadHeap::UploadPageInfo VulkanUploadHeap::CreateNewPage(VkDeviceSize SizeInBytes)
+{
+    VkBufferCreateInfo StagingBufferCI = {};
+    StagingBufferCI.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO;
+    StagingBufferCI.pNext = nullptr;
+    StagingBufferCI.flags = 0; // VK_BUFFER_CREATE_SPARSE_BINDING_BIT, VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT, VK_BUFFER_CREATE_SPARSE_ALIASED_BIT
+    StagingBufferCI.size  = SizeInBytes;
+    StagingBufferCI.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
+    StagingBufferCI.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
+    StagingBufferCI.queueFamilyIndexCount = 0;
+    StagingBufferCI.pQueueFamilyIndices = nullptr;
+
+    const auto& LogicalDevice  = m_RenderDevice.GetLogicalDevice();
+    const auto& PhysicalDevice = m_RenderDevice.GetPhysicalDevice();
+    auto& GlobalMemoryMgr = m_RenderDevice.GetGlobalMemoryManager();
+
+    auto NewBuffer = LogicalDevice.CreateBuffer(StagingBufferCI, "Upload buffer");
+    auto MemReqs = LogicalDevice.GetBufferMemoryRequirements(NewBuffer);
+    auto MemoryTypeIndex = PhysicalDevice.GetMemoryTypeIndex(MemReqs.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
+    VERIFY(MemoryTypeIndex != VulkanUtilities::VulkanPhysicalDevice::InvalidMemoryTypeIndex,
+           "Vulkan spec requires that for a VkBuffer not created with the VK_BUFFER_CREATE_SPARSE_BINDING_BIT "
+           "bit set, or for a VkImage that was created with a VK_IMAGE_TILING_LINEAR value in the tiling member "
+           "of the VkImageCreateInfo structure passed to vkCreateImage, the memoryTypeBits member always contains "
+           "at least one bit set corresponding to a VkMemoryType with a propertyFlags that has both the "
+           "VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT bit AND the VK_MEMORY_PROPERTY_HOST_COHERENT_BIT bit set. (11.6)");
+
+    auto MemAllocation = GlobalMemoryMgr.Allocate(MemReqs.size, MemReqs.alignment, MemoryTypeIndex, true);
+
+    auto AlignedOffset = (MemAllocation.UnalignedOffset + (MemReqs.alignment-1)) & ~(MemReqs.alignment-1);
+    auto err = LogicalDevice.BindBufferMemory(NewBuffer, MemAllocation.Page->GetVkMemory(), AlignedOffset);
+    CHECK_VK_ERROR_AND_THROW(err, "Failed to bind buffer memory");
+    auto CPUAddress = reinterpret_cast<Uint8*>(MemAllocation.Page->GetCPUMemory()) + AlignedOffset;
+
+    return UploadPageInfo{std::move(MemAllocation), std::move(NewBuffer), CPUAddress};
+}
+
+VulkanUploadAllocation VulkanUploadHeap::Allocate(size_t SizeInBytes)
+{
+    VulkanUploadAllocation Allocation;
+    if(SizeInBytes >= m_PageSize/2)
+    {
+        // Allocate large chunk directly from the memory manager
+        auto NewPage = CreateNewPage(SizeInBytes);
+        Allocation.vkBuffer   = NewPage.Buffer;
+        Allocation.CPUAddress = NewPage.CPUAddress;
+        Allocation.Size       = SizeInBytes;
+        Allocation.Offset     = 0;
+        m_CurrAllocatedSize   += NewPage.MemAllocation.Size;
+        m_Pages.emplace_back(std::move(NewPage));
+    }
+    else
+    {
+        if(m_CurrPage.AvailableSize < SizeInBytes)
+        {
+            auto NewPage = CreateNewPage(m_PageSize);
+            m_CurrPage.Reset(NewPage);
+            VERIFY_EXPR(m_CurrPage.AvailableSize == m_PageSize);
+            m_CurrAllocatedSize += NewPage.MemAllocation.Size;
+            m_Pages.emplace_back(std::move(NewPage));
+        }
+
+        Allocation.vkBuffer   = m_CurrPage.vkBuffer;
+        Allocation.CPUAddress = m_CurrPage.CPUAddress;
+        Allocation.Size       = SizeInBytes;
+        Allocation.Offset     = m_CurrPage.CurrOffset;
+        m_CurrPage.Advance(SizeInBytes);
+    }
+    m_CurrFrameSize += SizeInBytes;
+    m_PeakFrameSize     = std::max(m_CurrFrameSize, m_PeakFrameSize);
+    m_PeakAllocatedSize = std::max(m_CurrAllocatedSize, m_PeakAllocatedSize);
+
+    return Allocation;
+}
+
+void VulkanUploadHeap::DiscardAllocations(uint64_t FenceValue)
+{
+    auto& ReleaseQueue = m_RenderDevice.GetReleaseQueue();
+
+    {
+        auto AllocIt = m_Pages.begin();
+        ReleaseQueue.DiscardResources<VulkanUtilities::VulkanMemoryAllocation>(FenceValue, [&](VulkanUtilities::VulkanMemoryAllocation& MemAllocation)
+        {
+            if(AllocIt != m_Pages.end())
+            {
+                MemAllocation = std::move(AllocIt->MemAllocation);
+                ++AllocIt;
+                return true;
+            }
+            return false;
+        });
+    }
+
+    {
+        auto AllocIt = m_Pages.begin();
+        ReleaseQueue.DiscardResources<VulkanUtilities::BufferWrapper>(FenceValue, [&](VulkanUtilities::BufferWrapper& Buffer)
+        {
+            if(AllocIt != m_Pages.end())
+            {
+                Buffer = std::move(AllocIt->Buffer);
+                ++AllocIt;
+                return true;
+            }
+            return false;
+        });
+    }
+
+    m_Pages.clear();
+
+    m_CurrPage = CurrPageInfo{};
+    m_CurrFrameSize     = 0;
+    m_CurrAllocatedSize = 0;
+}
+
+}
diff --git a/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanUploadHeap.cpp b/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanUploadHeap.cpp
deleted file mode 100644
index ad6e9fbc..00000000
--- a/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanUploadHeap.cpp
+++ /dev/null
@@ -1,104 +0,0 @@
-/*     Copyright 2015-2018 Egor Yusov
- *  
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF ANY PROPRIETARY RIGHTS.
- *
- *  In no event and under no legal theory, whether in tort (including negligence), 
- *  contract, or otherwise, unless required by applicable law (such as deliberate 
- *  and grossly negligent acts) or agreed to in writing, shall any Contributor be
- *  liable for any damages, including any direct, indirect, special, incidental, 
- *  or consequential damages of any character arising as a result of this License or 
- *  out of the use or inability to use the software (including but not limited to damages 
- *  for loss of goodwill, work stoppage, computer failure or malfunction, or any and 
- *  all other commercial damages or losses), even if such Contributor has been advised 
- *  of the possibility of such damages.
- */
-
-#include "pch.h"
-#include "VulkanUtilities/VulkanUploadHeap.h"
-
-namespace VulkanUtilities
-{
-
-VkBufferCreateInfo VulkanUploadHeap::GetStagingBufferCI()const
-{
-    VkBufferCreateInfo StagingBufferCI = {};
-    StagingBufferCI.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO;
-    StagingBufferCI.pNext = nullptr;
-    StagingBufferCI.flags = 0; // VK_BUFFER_CREATE_SPARSE_BINDING_BIT, VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT, VK_BUFFER_CREATE_SPARSE_ALIASED_BIT
-    StagingBufferCI.size  = m_HostVisiblePageSize;
-    StagingBufferCI.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
-    StagingBufferCI.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
-    StagingBufferCI.queueFamilyIndexCount = 0;
-    StagingBufferCI.pQueueFamilyIndices = nullptr;
-    return StagingBufferCI;
-}
-
-VulkanUploadHeap::VulkanUploadHeap(std::string                  MgrName,
-                                   const VulkanLogicalDevice&   LogicalDevice, 
-                                   const VulkanPhysicalDevice&  PhysicalDevice, 
-                                   Diligent::IMemoryAllocator&  Allocator, 
-                                   VkDeviceSize                 HostVisiblePageSize,
-                                   VkDeviceSize                 HostVisibleReserveSize) :
-    VulkanMemoryManager(MgrName, LogicalDevice, PhysicalDevice, Allocator, 0, HostVisiblePageSize, 0, HostVisibleReserveSize)
-{
-    auto StagingBufferCI = GetStagingBufferCI();
-    auto TmpStagingBuffer = LogicalDevice.CreateBuffer(StagingBufferCI);
-
-    VkMemoryRequirements StagingBufferMemReqs = LogicalDevice.GetBufferMemoryRequirements(TmpStagingBuffer);
-    m_StagingBufferMemoryTypeIndex = m_PhysicalDevice.GetMemoryTypeIndex(StagingBufferMemReqs.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
-    VERIFY(m_StagingBufferMemoryTypeIndex != VulkanUtilities::VulkanPhysicalDevice::InvalidMemoryTypeIndex,
-           "Vulkan spec requires that for a VkBuffer not created with the VK_BUFFER_CREATE_SPARSE_BINDING_BIT "
-           "bit set, or for a VkImage that was created with a VK_IMAGE_TILING_LINEAR value in the tiling member "
-           "of the VkImageCreateInfo structure passed to vkCreateImage, the memoryTypeBits member always contains "
-           "at least one bit set corresponding to a VkMemoryType with a propertyFlags that has both the "
-           "VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT bit AND the VK_MEMORY_PROPERTY_HOST_COHERENT_BIT bit set. (11.6)");
-}
-
-VulkanUploadHeap::~VulkanUploadHeap()
-{
-    VERIFY_EXPR(m_Pages.size() == m_Buffers.size());
-}
-
-VulkanUploadAllocation VulkanUploadHeap::Allocate(size_t SizeInBytes)
-{
-    VulkanUploadAllocation Allocation;
-    Allocation.MemAllocation = VulkanMemoryManager::Allocate(SizeInBytes, 0, m_StagingBufferMemoryTypeIndex, true);
-
-    std::lock_guard<std::mutex> Lock(m_BuffersMtx);
-    auto BuffIt = m_Buffers.find(Allocation.MemAllocation.Page);
-    VERIFY_EXPR(BuffIt != m_Buffers.end());
-    Allocation.vkBuffer = BuffIt->second;
-
-    return Allocation;
-}
-
-void VulkanUploadHeap::OnNewPageCreated(VulkanMemoryPage& NewPage)
-{
-    std::lock_guard<std::mutex> Lock(m_BuffersMtx);
-    auto BufferCI = GetStagingBufferCI();
-    auto NewBuffer = m_LogicalDevice.CreateBuffer(BufferCI, "Upload buffer");
-    auto MemReqs = m_LogicalDevice.GetBufferMemoryRequirements(NewBuffer);
-    auto MemoryTypeIndex = m_PhysicalDevice.GetMemoryTypeIndex(MemReqs.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
-    VERIFY(MemoryTypeIndex == m_StagingBufferMemoryTypeIndex, "Incosistent memory type");
-    auto err = m_LogicalDevice.BindBufferMemory(NewBuffer, NewPage.GetVkMemory(), 0);
-    CHECK_VK_ERROR_AND_THROW(err, "Failed to bind buffer memory");
-    VERIFY(m_Buffers.find(&NewPage) == m_Buffers.end(), "Buffer corresponding to this page already exists");
-    m_Buffers.emplace(&NewPage, std::move(NewBuffer));
-}
-
-void VulkanUploadHeap::OnPageDestroy(VulkanMemoryPage& Page)
-{
-    std::lock_guard<std::mutex> Lock(m_BuffersMtx);
-    auto ElemsRemoved = m_Buffers.erase(&Page);
-    VERIFY_EXPR(ElemsRemoved == 1);
-}
-
-}