diff options
| author | Egor Yusov <egor.yusov@gmail.com> | 2018-06-04 00:50:39 +0000 |
|---|---|---|
| committer | Egor Yusov <egor.yusov@gmail.com> | 2018-06-04 00:50:39 +0000 |
| commit | 934176e4de04f380ed3fed77588d396b41477bbc (patch) | |
| tree | dace42151093e337c9a4683295d26bdda8a62807 /Graphics/GraphicsEngineVulkan | |
| parent | Fixed issue with partial views of a 3D texture in Vulkan (diff) | |
| download | DiligentCore-934176e4de04f380ed3fed77588d396b41477bbc.tar.gz DiligentCore-934176e4de04f380ed3fed77588d396b41477bbc.zip | |
Implemented Vulkan memory manager
Diffstat (limited to 'Graphics/GraphicsEngineVulkan')
11 files changed, 501 insertions, 164 deletions
diff --git a/Graphics/GraphicsEngineVulkan/CMakeLists.txt b/Graphics/GraphicsEngineVulkan/CMakeLists.txt index fb574032..31502969 100644 --- a/Graphics/GraphicsEngineVulkan/CMakeLists.txt +++ b/Graphics/GraphicsEngineVulkan/CMakeLists.txt @@ -9,7 +9,6 @@ set(INCLUDE include/CommandListVkImpl.h include/CommandPoolManager.h include/CommandQueueVkImpl.h - include/VulkanTypeConversions.h include/DescriptorPoolManager.h include/DeviceContextVkImpl.h include/VulkanDynamicHeap.h @@ -28,6 +27,7 @@ set(INCLUDE include/SwapChainVkImpl.h include/TextureVkImpl.h include/TextureViewVkImpl.h + include/VulkanTypeConversions.h include/VulkanErrors.h ) @@ -39,6 +39,7 @@ set(VULKAN_UTILS_INCLUDE include/VulkanUtilities/VulkanFencePool.h include/VulkanUtilities/VulkanInstance.h include/VulkanUtilities/VulkanLogicalDevice.h + include/VulkanUtilities/VulkanMemoryManager.h include/VulkanUtilities/VulkanObjectWrappers.h include/VulkanUtilities/VulkanPhysicalDevice.h ) @@ -67,7 +68,6 @@ set(SRC src/CommandContext.cpp src/CommandPoolManager.cpp src/CommandQueueVkImpl.cpp - src/VulkanTypeConversions.cpp src/DescriptorPoolManager.cpp src/DeviceContextVkImpl.cpp src/VulkanDynamicHeap.cpp @@ -86,6 +86,7 @@ set(SRC src/SwapChainVkImpl.cpp src/TextureVkImpl.cpp src/TextureViewVkImpl.cpp + src/VulkanTypeConversions.cpp ) set(VULKAN_UTILS_SRC @@ -96,6 +97,7 @@ set(VULKAN_UTILS_SRC src/VulkanUtilities/VulkanFencePool.cpp src/VulkanUtilities/VulkanInstance.cpp src/VulkanUtilities/VulkanLogicalDevice.cpp + src/VulkanUtilities/VulkanMemoryManager.cpp src/VulkanUtilities/VulkanPhysicalDevice.cpp ) diff --git a/Graphics/GraphicsEngineVulkan/include/BufferVkImpl.h b/Graphics/GraphicsEngineVulkan/include/BufferVkImpl.h index c35863f0..1ac1e384 100644 --- a/Graphics/GraphicsEngineVulkan/include/BufferVkImpl.h +++ b/Graphics/GraphicsEngineVulkan/include/BufferVkImpl.h @@ -32,6 +32,7 @@ #include "BufferViewVkImpl.h" #include "VulkanDynamicHeap.h" #include "VulkanUtilities/VulkanObjectWrappers.h" +#include "VulkanUtilities/VulkanMemoryManager.h" namespace Diligent { @@ -94,7 +95,7 @@ private: #endif VulkanUtilities::BufferWrapper m_VulkanBuffer; - VulkanUtilities::DeviceMemoryWrapper m_BufferMemory; + VulkanUtilities::VulkanMemoryAllocation m_MemoryAllocation; }; } diff --git a/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h b/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h index 8946e694..fb816379 100644 --- a/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h +++ b/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h @@ -39,6 +39,7 @@ #include "VulkanUtilities/VulkanCommandBufferPool.h" #include "VulkanUtilities/VulkanLogicalDevice.h" #include "VulkanUtilities/VulkanObjectWrappers.h" +#include "VulkanUtilities/VulkanMemoryManager.h" #include "FramebufferCache.h" #include "CommandPoolManager.h" @@ -103,6 +104,7 @@ public: template<typename VulkanObjectType> void SafeReleaseVkObject(VulkanUtilities::VulkanObjectWrapper<VulkanObjectType>&& vkObject); + void SafeReleaseMemoryAllocation(VulkanUtilities::VulkanMemoryAllocation&& Allocation); void FinishFrame(bool ReleaseAllResources); @@ -129,6 +131,11 @@ public: const auto &GetLogicalDevice(){return *m_LogicalVkDevice;} FramebufferCache& GetFramebufferCache(){return m_FramebufferCache;} + VulkanUtilities::VulkanMemoryAllocation AllocateMemory(const VkMemoryRequirements& MemReqs, VkMemoryPropertyFlags MemoryProperties) + { + return m_MemoryMgr.Allocate(MemReqs, MemoryProperties); + } + private: virtual void TestTextureFormat( TEXTURE_FORMAT TexFormat )override final; void ProcessReleaseQueue(Uint64 CompletedFenceValue); @@ -188,8 +195,6 @@ private: public: virtual ~StaleVulkanObjectBase() = 0 {} }; - template<typename VulkanObjectType> - class StaleVulkanObject; using ReleaseQueueElemType = std::pair<Uint64, std::unique_ptr<StaleVulkanObjectBase> >; std::deque< ReleaseQueueElemType, STDAllocatorRawMem<ReleaseQueueElemType> > m_VkObjReleaseQueue; @@ -210,6 +215,8 @@ private: // issue copy commands. Vulkan requires that every command pool is used by one thread // at a time, so every constructor must allocate command buffer from its own pool. CommandPoolManager m_TransientCmdPoolMgr; + + VulkanUtilities::VulkanMemoryManager m_MemoryMgr; }; } diff --git a/Graphics/GraphicsEngineVulkan/include/TextureVkImpl.h b/Graphics/GraphicsEngineVulkan/include/TextureVkImpl.h index d08f7084..293e0966 100644 --- a/Graphics/GraphicsEngineVulkan/include/TextureVkImpl.h +++ b/Graphics/GraphicsEngineVulkan/include/TextureVkImpl.h @@ -30,6 +30,7 @@ #include "RenderDeviceVk.h" #include "TextureBase.h" #include "TextureViewVkImpl.h" +#include "VulkanUtilities/VulkanMemoryManager.h" namespace Diligent { @@ -112,8 +113,9 @@ protected: friend class RenderDeviceVkImpl; */ + VulkanUtilities::ImageWrapper m_VulkanImage; - VulkanUtilities::DeviceMemoryWrapper m_ImageMemory; + VulkanUtilities::VulkanMemoryAllocation m_MemoryAllocation; VkImageLayout m_CurrentLayout = VK_IMAGE_LAYOUT_UNDEFINED; }; diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanMemoryManager.h b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanMemoryManager.h new file mode 100644 index 00000000..e8679ad9 --- /dev/null +++ b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanMemoryManager.h @@ -0,0 +1,184 @@ +/* Copyright 2015-2018 Egor Yusov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF ANY PROPRIETARY RIGHTS. + * + * In no event and under no legal theory, whether in tort (including negligence), + * contract, or otherwise, unless required by applicable law (such as deliberate + * and grossly negligent acts) or agreed to in writing, shall any Contributor be + * liable for any damages, including any direct, indirect, special, incidental, + * or consequential damages of any character arising as a result of this License or + * out of the use or inability to use the software (including but not limited to damages + * for loss of goodwill, work stoppage, computer failure or malfunction, or any and + * all other commercial damages or losses), even if such Contributor has been advised + * of the possibility of such damages. + */ + +#pragma once + +#include <mutex> +#include <array> +#include <unordered_map> +#include <atomic> +#include <string> +#include "MemoryAllocator.h" +#include "VariableSizeAllocationsManager.h" +#include "VulkanUtilities/VulkanPhysicalDevice.h" +#include "VulkanUtilities/VulkanLogicalDevice.h" +#include "VulkanUtilities/VulkanObjectWrappers.h" + +namespace VulkanUtilities +{ + +class VulkanMemoryPage; +class VulkanMemoryManager; + +struct VulkanMemoryAllocation +{ + VulkanMemoryAllocation()noexcept{} + + VulkanMemoryAllocation (const VulkanMemoryAllocation&) = delete; + VulkanMemoryAllocation& operator= (const VulkanMemoryAllocation&) = delete; + + VulkanMemoryAllocation(VulkanMemoryPage* _Page, size_t _UnalignedOffset, size_t _Size)noexcept : + Page (_Page), + UnalignedOffset(_UnalignedOffset), + Size (_Size) + {} + + VulkanMemoryAllocation(VulkanMemoryAllocation&& rhs)noexcept : + Page (rhs.Page), + UnalignedOffset(rhs.UnalignedOffset), + Size (rhs.Size) + { + rhs.Page = nullptr; + rhs.UnalignedOffset = 0; + rhs.Size = 0; + } + + VulkanMemoryAllocation& operator= (VulkanMemoryAllocation&& rhs)noexcept + { + Page = rhs.Page; + UnalignedOffset = rhs.UnalignedOffset; + Size = rhs.Size; + + rhs.Page = nullptr; + rhs.UnalignedOffset = 0; + rhs.Size = 0; + + return *this; + } + + // Destructor immediately returns the allocation to the parent page. + // The allocation must not be in use by the GPU. + ~VulkanMemoryAllocation(); + + VulkanMemoryPage* Page = nullptr; // Memory page that contains this allocation + size_t UnalignedOffset = 0; // Unaligned offset from the start of the memory + size_t Size = 0; // Reserved size of this allocation +}; + +class VulkanMemoryPage +{ +public: + VulkanMemoryPage(VulkanMemoryManager& ParentMemoryMgr, + VkDeviceSize PageSize, + uint32_t MemoryTypeIndex, + bool MapMemory)noexcept; + ~VulkanMemoryPage(); + + VulkanMemoryPage(VulkanMemoryPage&& rhs)noexcept : + m_ParentMemoryMgr (rhs.m_ParentMemoryMgr), + m_AllocationMgr (std::move(rhs.m_AllocationMgr)), + m_VkMemory (std::move(rhs.m_VkMemory)), + m_CPUMemory (rhs.m_CPUMemory) + { + rhs.m_CPUMemory = nullptr; + } + + VulkanMemoryPage (const VulkanMemoryPage&) = delete; + VulkanMemoryPage& operator= (VulkanMemoryPage&) = delete; + VulkanMemoryPage& operator= (VulkanMemoryPage&& rhs) = delete; + + bool IsEmpty()const{return m_AllocationMgr.IsEmpty();} + bool IsFull() const{return m_AllocationMgr.IsFull();} + VkDeviceSize GetPageSize()const{return m_AllocationMgr.GetMaxSize();} + VkDeviceSize GetUsedSize()const{return m_AllocationMgr.GetUsedSize();} + + VulkanMemoryAllocation Allocate(VkDeviceSize size); + + VkDeviceMemory GetVkMemory()const{return m_VkMemory;} + void* GetCPUMemory()const{return m_CPUMemory;} + +private: + friend struct VulkanMemoryAllocation; + + // Memory is reclaimed immediately. The application is responsible to ensure it is not in use by the GPU + void Free(VulkanMemoryAllocation& Allocation); + + VulkanMemoryManager& m_ParentMemoryMgr; + std::mutex m_Mutex; + Diligent::VariableSizeAllocationsManager m_AllocationMgr; + VulkanUtilities::DeviceMemoryWrapper m_VkMemory; + void* m_CPUMemory = nullptr; +}; + +class VulkanMemoryManager +{ +public: + VulkanMemoryManager(std::string MgrName, + const VulkanLogicalDevice& LogicalDevice, + const VulkanPhysicalDevice& PhysicalDevice, + Diligent::IMemoryAllocator& Allocator, + VkDeviceSize DeviceLocalPageSize, + VkDeviceSize HostVisiblePageSize) : + m_MgrName (std::move(MgrName)), + m_LogicalDevice (LogicalDevice), + m_PhysicalDevice (PhysicalDevice), + m_Allocator (Allocator), + m_DeviceLocalPageSize(DeviceLocalPageSize), + m_HostVisiblePageSize(HostVisiblePageSize) + {} + + ~VulkanMemoryManager(); + + VulkanMemoryManager (const VulkanMemoryManager&) = delete; + VulkanMemoryManager (VulkanMemoryManager&&) = delete; + VulkanMemoryManager& operator= (const VulkanMemoryManager&) = delete; + VulkanMemoryManager& operator= (VulkanMemoryManager&&) = delete; + + VulkanMemoryAllocation Allocate(const VkMemoryRequirements& MemReqs, VkMemoryPropertyFlags MemoryProps); + void ShrinkMemory(); + +private: + friend class VulkanMemoryPage; + + const std::string m_MgrName; + + const VulkanLogicalDevice& m_LogicalDevice; + const VulkanPhysicalDevice& m_PhysicalDevice; + + Diligent::IMemoryAllocator& m_Allocator; + + std::unordered_multimap<uint32_t, VulkanMemoryPage> m_Pages; + std::mutex m_Mutex; + const VkDeviceSize m_DeviceLocalPageSize; + const VkDeviceSize m_HostVisiblePageSize; + + void OnFreeAllocation(VkDeviceSize Size, bool IsHostVisble); + + // 0 == Device local, 1 == Host-visible + std::array<std::atomic_int64_t, 2> m_CurrUsedSize = {}; + std::array<VkDeviceSize, 2> m_PeakUsedSize = {}; + std::array<VkDeviceSize, 2> m_CurrAllocatedSize = {}; + std::array<VkDeviceSize, 2> m_PeakAllocatedSize = {}; +}; + +} diff --git a/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp index 61e3d910..a1711629 100644 --- a/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp +++ b/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp @@ -99,50 +99,21 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters *pRefCounters, VkMemoryRequirements MemReqs = LogicalDevice.GetBufferMemoryRequirements(m_VulkanBuffer); - VkMemoryAllocateInfo MemAlloc = {}; - MemAlloc.pNext = nullptr; - MemAlloc.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; - MemAlloc.allocationSize = MemReqs.size; - - auto& PhysicalDevice = pRenderDeviceVk->GetPhysicalDevice(); - VkMemoryPropertyFlags BufferMemoryFlags = 0; if (m_Desc.Usage == USAGE_CPU_ACCESSIBLE) BufferMemoryFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; else BufferMemoryFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - // memoryTypeBits is a bitmask and contains one bit set for every supported memory type for the resource. - // Bit i is set if and only if the memory type i in the VkPhysicalDeviceMemoryProperties structure for the - // physical device is supported for the resource. - MemAlloc.memoryTypeIndex = PhysicalDevice.GetMemoryTypeIndex(MemReqs.memoryTypeBits, BufferMemoryFlags); - if(BufferMemoryFlags == VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) - { - // There must be at least one memory type with the DEVICE_LOCAL_BIT bit set - VERIFY(MemAlloc.memoryTypeIndex != VulkanUtilities::VulkanPhysicalDevice::InvalidMemoryTypeIndex, - "Vulkan spec requires that memoryTypeBits member always contains " - "at least one bit set corresponding to a VkMemoryType with a propertyFlags that has the " - "VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT bit set (11.6)"); - } - else if(MemAlloc.memoryTypeIndex == VulkanUtilities::VulkanPhysicalDevice::InvalidMemoryTypeIndex) - { - LOG_ERROR_AND_THROW("Failed to find suitable device memory type for a buffer"); - } - - { - std::string MemoryName("Device local memory for buffer '"); - MemoryName += m_Desc.Name; - MemoryName += '\''; - m_BufferMemory = LogicalDevice.AllocateDeviceMemory(MemAlloc, MemoryName.c_str()); - } + m_MemoryAllocation = pRenderDeviceVk->AllocateMemory(MemReqs, BufferMemoryFlags); - auto err = LogicalDevice.BindBufferMemory(m_VulkanBuffer, m_BufferMemory, 0 /*offset*/); + VERIFY( (MemReqs.alignment & (MemReqs.alignment-1)) == 0, "Alignment is not power of 2!"); + auto AlignedOffset = (m_MemoryAllocation.UnalignedOffset + (MemReqs.alignment-1)) & ~(MemReqs.alignment-1); + auto Memory = m_MemoryAllocation.Page->GetVkMemory(); + auto err = LogicalDevice.BindBufferMemory(m_VulkanBuffer, Memory, AlignedOffset); CHECK_VK_ERROR_AND_THROW(err, "Failed to bind buffer memory"); bool bInitializeBuffer = (BuffData.pData != nullptr && BuffData.DataSize > 0); - //if(bInitializeBuffer) - // m_UsageState = Vk_RESOURCE_STATE_COPY_DEST; - if( bInitializeBuffer ) { VkBufferCreateInfo VkStaginBuffCI = VkBuffCI; @@ -155,40 +126,18 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters *pRefCounters, VkMemoryRequirements StagingBufferMemReqs = LogicalDevice.GetBufferMemoryRequirements(StagingBuffer); - VkMemoryAllocateInfo StagingMemAlloc = {}; - StagingMemAlloc.pNext = nullptr; - StagingMemAlloc.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; - StagingMemAlloc.allocationSize = StagingBufferMemReqs.size; - // VK_MEMORY_PROPERTY_HOST_COHERENT_BIT bit specifies that the host cache management commands vkFlushMappedMemoryRanges // and vkInvalidateMappedMemoryRanges are NOT needed to flush host writes to the device or make device writes visible // to the host (10.2) - StagingMemAlloc.memoryTypeIndex = PhysicalDevice.GetMemoryTypeIndex(StagingBufferMemReqs.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); - - VERIFY(StagingMemAlloc.memoryTypeIndex != VulkanUtilities::VulkanPhysicalDevice::InvalidMemoryTypeIndex, - "Vulkan spec requires that for a VkBuffer not created with the " - "VK_BUFFER_CREATE_SPARSE_BINDING_BIT bit set, the memoryTypeBits member always contains at least one bit set " - "corresponding to a VkMemoryType with a propertyFlags that has both the VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT bit " - "and the VK_MEMORY_PROPERTY_HOST_COHERENT_BIT bit set(11.6)"); + auto StagingMemoryAllocation = pRenderDeviceVk->AllocateMemory(StagingBufferMemReqs, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + auto StagingBufferMemory = StagingMemoryAllocation.Page->GetVkMemory(); + auto AlignedStagingMemOffset = (StagingMemoryAllocation.UnalignedOffset + (StagingBufferMemReqs.alignment-1)) & ~(StagingBufferMemReqs.alignment-1); - std::string StagingMemoryName("Staging memory for buffer '"); - StagingMemoryName += m_Desc.Name; - StagingMemoryName += '\''; - VulkanUtilities::DeviceMemoryWrapper StagingBufferMemory = LogicalDevice.AllocateDeviceMemory(StagingMemAlloc, StagingMemoryName.c_str()); - - { - void *StagingData = nullptr; - err = LogicalDevice.MapMemory(StagingBufferMemory, - 0, // offset - StagingMemAlloc.allocationSize, - 0, // flags, reserved for future use - &StagingData); - CHECK_VK_ERROR_AND_THROW(err, "Failed to map staging memory"); - memcpy(StagingData, BuffData.pData, BuffData.DataSize); - LogicalDevice.UnmapMemory(StagingBufferMemory); - } + auto *StagingData = reinterpret_cast<uint8_t*>(StagingMemoryAllocation.Page->GetCPUMemory()); + VERIFY_EXPR(StagingData != nullptr); + memcpy(StagingData + AlignedStagingMemOffset, BuffData.pData, BuffData.DataSize); - err = LogicalDevice.BindBufferMemory(StagingBuffer, StagingBufferMemory, 0 /*offset*/); + err = LogicalDevice.BindBufferMemory(StagingBuffer, StagingBufferMemory, AlignedStagingMemOffset); CHECK_VK_ERROR_AND_THROW(err, "Failed to bind staging bufer memory"); VulkanUtilities::CommandPoolWrapper CmdPool; @@ -233,7 +182,7 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters *pRefCounters, // until copy operation is complete. This must be done after // submitting command list for execution! pRenderDeviceVk->SafeReleaseVkObject(std::move(StagingBuffer)); - pRenderDeviceVk->SafeReleaseVkObject(std::move(StagingBufferMemory)); + pRenderDeviceVk->SafeReleaseMemoryAllocation(std::move(StagingMemoryAllocation)); } else { @@ -310,7 +259,7 @@ BufferVkImpl :: ~BufferVkImpl() auto *pDeviceVkImpl = ValidatedCast<RenderDeviceVkImpl>(GetDevice()); // Vk object can only be destroyed when it is no longer used by the GPU pDeviceVkImpl->SafeReleaseVkObject(std::move(m_VulkanBuffer)); - pDeviceVkImpl->SafeReleaseVkObject(std::move(m_BufferMemory)); + pDeviceVkImpl->SafeReleaseMemoryAllocation(std::move(m_MemoryAllocation)); } IMPLEMENT_QUERY_INTERFACE( BufferVkImpl, IID_BufferVk, TBufferBase ) diff --git a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp index 9a27894f..631f5542 100644 --- a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp +++ b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp @@ -758,7 +758,8 @@ namespace Diligent SubmitInfo.signalSemaphoreCount = static_cast<uint32_t>(m_SignalSemaphores.size()); SubmitInfo.pSignalSemaphores = SubmitInfo.signalSemaphoreCount != 0 ? m_SignalSemaphores.data() : nullptr; - if(SubmitInfo.commandBufferCount != 0 || SubmitInfo.waitSemaphoreCount !=0 || SubmitInfo.signalSemaphoreCount != 0) + // Submit command buffer even if there are no commands to release stale resources. + //if(SubmitInfo.commandBufferCount != 0 || SubmitInfo.waitSemaphoreCount !=0 || SubmitInfo.signalSemaphoreCount != 0) { pDeviceVkImpl->ExecuteCommandBuffer(SubmitInfo, true); } diff --git a/Graphics/GraphicsEngineVulkan/src/RenderDeviceFactoryVk.cpp b/Graphics/GraphicsEngineVulkan/src/RenderDeviceFactoryVk.cpp index 86e38add..7bc2d843 100644 --- a/Graphics/GraphicsEngineVulkan/src/RenderDeviceFactoryVk.cpp +++ b/Graphics/GraphicsEngineVulkan/src/RenderDeviceFactoryVk.cpp @@ -150,15 +150,17 @@ void EngineFactoryVkImpl::CreateDeviceAndContextsVk( const EngineVkAttribs& Crea DeviceCreateInfo.queueCreateInfoCount = 1; DeviceCreateInfo.pQueueCreateInfos = &QueueInfo; VkPhysicalDeviceFeatures DeviceFeatures = {}; - DeviceFeatures.depthBiasClamp = VK_TRUE; - DeviceFeatures.fillModeNonSolid = VK_TRUE; - DeviceFeatures.depthClamp = VK_TRUE; - DeviceFeatures.independentBlend = VK_TRUE; - DeviceFeatures.samplerAnisotropy = VK_TRUE; - DeviceFeatures.geometryShader = VK_TRUE; - DeviceFeatures.tessellationShader= VK_TRUE; - DeviceFeatures.dualSrcBlend = VK_TRUE; - DeviceFeatures.multiViewport = VK_TRUE; + DeviceFeatures.depthBiasClamp = VK_TRUE; + DeviceFeatures.fillModeNonSolid = VK_TRUE; + DeviceFeatures.depthClamp = VK_TRUE; + DeviceFeatures.independentBlend = VK_TRUE; + DeviceFeatures.samplerAnisotropy = VK_TRUE; + DeviceFeatures.geometryShader = VK_TRUE; + DeviceFeatures.tessellationShader = VK_TRUE; + DeviceFeatures.dualSrcBlend = VK_TRUE; + DeviceFeatures.multiViewport = VK_TRUE; + DeviceFeatures.imageCubeArray = VK_TRUE; + DeviceFeatures.textureCompressionBC = VK_TRUE; DeviceCreateInfo.pEnabledFeatures = &DeviceFeatures; // NULL or a pointer to a VkPhysicalDeviceFeatures structure that contains // boolean indicators of all the features to be enabled. diff --git a/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp index 5744329c..60b3df85 100644 --- a/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp +++ b/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp @@ -47,7 +47,7 @@ RenderDeviceVkImpl :: RenderDeviceVkImpl(IReferenceCounters *pRefCounters, TRenderDeviceBase(pRefCounters, RawMemAllocator, NumDeferredContexts, sizeof(TextureVkImpl), sizeof(TextureViewVkImpl), sizeof(BufferVkImpl), sizeof(BufferViewVkImpl), sizeof(ShaderVkImpl), sizeof(SamplerVkImpl), sizeof(PipelineStateVkImpl), sizeof(ShaderResourceBindingVkImpl)), m_VulkanInstance(Instance), m_PhysicalDevice(std::move(PhysicalDevice)), - m_LogicalVkDevice(LogicalDevice), + m_LogicalVkDevice(std::move(LogicalDevice)), m_pCommandQueue(pCmdQueue), m_EngineAttribs(CreationAttribs), m_FrameNumber(0), @@ -60,7 +60,8 @@ RenderDeviceVkImpl :: RenderDeviceVkImpl(IReferenceCounters *pRefCounters, m_DescriptorPools(STD_ALLOCATOR_RAW_MEM(DescriptorPoolManager, GetRawAllocator(), "Allocator for vector<DescriptorPoolManager>")), m_UploadHeaps(STD_ALLOCATOR_RAW_MEM(UploadHeapPoolElemType, GetRawAllocator(), "Allocator for vector<unique_ptr<VulkanDynamicHeap>>")), m_FramebufferCache(*this), - m_TransientCmdPoolMgr(*LogicalDevice, pCmdQueue->GetQueueFamilyIndex(), VK_COMMAND_POOL_CREATE_TRANSIENT_BIT) + m_TransientCmdPoolMgr(*m_LogicalVkDevice, pCmdQueue->GetQueueFamilyIndex(), VK_COMMAND_POOL_CREATE_TRANSIENT_BIT), + m_MemoryMgr("Global resource memory manager", *m_LogicalVkDevice, *m_PhysicalDevice, GetRawAllocator(), CreationAttribs.DeviceLocalMemoryPageSize, CreationAttribs.HostVisibleMemoryPageSize ) { m_DeviceCaps.DevType = DeviceType::Vulkan; m_DeviceCaps.MajorVersion = 1; @@ -72,7 +73,7 @@ RenderDeviceVkImpl :: RenderDeviceVkImpl(IReferenceCounters *pRefCounters, m_DescriptorPools.reserve(2 + NumDeferredContexts); m_DescriptorPools.emplace_back( - LogicalDevice, + m_LogicalVkDevice, std::vector<VkDescriptorPoolSize>{ {VK_DESCRIPTOR_TYPE_SAMPLER, CreationAttribs.MainDescriptorPoolSize.NumSeparateSamplerDescriptors}, {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, CreationAttribs.MainDescriptorPoolSize.NumCombinedSamplerDescriptors}, @@ -93,7 +94,7 @@ RenderDeviceVkImpl :: RenderDeviceVkImpl(IReferenceCounters *pRefCounters, for(Uint32 ctx = 0; ctx < 1 + NumDeferredContexts; ++ctx) { m_DescriptorPools.emplace_back( - LogicalDevice, + m_LogicalVkDevice, std::vector<VkDescriptorPoolSize>{ {VK_DESCRIPTOR_TYPE_SAMPLER, CreationAttribs.DynamicDescriptorPoolSize.NumSeparateSamplerDescriptors}, {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, CreationAttribs.DynamicDescriptorPoolSize.NumCombinedSamplerDescriptors}, @@ -213,6 +214,9 @@ void RenderDeviceVkImpl::ExecuteCommandBuffer(const VkSubmitInfo &SubmitInfo, bo // is met even when the fence value is not incremented while executing // the command list (as is the case with Unity command queue). DiscardStaleVkObjects(CmdListNumber, FenceValue); + auto CompletedFenceValue = GetCompletedFenceValue(); + ProcessReleaseQueue(CompletedFenceValue); + m_MemoryMgr.ShrinkMemory(); } #if 0 // DiscardAllocator() is thread-safe @@ -255,7 +259,9 @@ void RenderDeviceVkImpl::IdleGPU(bool ReleaseStaleObjects) // swap chain buffers when it is resized in the middle of the frame. // Since GPU has been idled, it it is safe to do so DiscardStaleVkObjects(CmdListNumber, FenceValue); + // FenceValue has now been signaled by the GPU since we waited for it ProcessReleaseQueue(FenceValue); + m_MemoryMgr.ShrinkMemory(); } } @@ -352,6 +358,7 @@ void RenderDeviceVkImpl::FinishFrame(bool ReleaseAllResources) // no command lists submitted during the frame DiscardStaleVkObjects(CmdListNumber, NextFenceValue); ProcessReleaseQueue(CompletedFenceValue); + m_MemoryMgr.ShrinkMemory(); Atomics::AtomicIncrement(m_FrameNumber); } @@ -393,30 +400,29 @@ void RenderDeviceVkImpl::DisposeTransientCmdPool(VulkanUtilities::CommandPoolWra template<typename VulkanObjectType> -class RenderDeviceVkImpl::StaleVulkanObject : public StaleVulkanObjectBase +void RenderDeviceVkImpl::SafeReleaseVkObject(VulkanUtilities::VulkanObjectWrapper<VulkanObjectType>&& vkObject) { -public: - StaleVulkanObject(VulkanUtilities::VulkanObjectWrapper<VulkanObjectType> &&Object) : - m_VkObject(std::move(Object)) - {} - - ~StaleVulkanObject() + class StaleVulkanObject : public RenderDeviceVkImpl::StaleVulkanObjectBase { - m_VkObject.Release(); - } + public: + StaleVulkanObject(VulkanUtilities::VulkanObjectWrapper<VulkanObjectType>&& Object) : + m_VkObject(std::move(Object)) + {} -private: - VulkanUtilities::VulkanObjectWrapper<VulkanObjectType> m_VkObject; -}; + ~StaleVulkanObject() + { + m_VkObject.Release(); + } + + private: + VulkanUtilities::VulkanObjectWrapper<VulkanObjectType> m_VkObject; + }; -template<typename VulkanObjectType> -void RenderDeviceVkImpl::SafeReleaseVkObject(VulkanUtilities::VulkanObjectWrapper<VulkanObjectType>&& vkObject) -{ // When Vk object is released, it is first moved into the // stale objects list. The list is moved into a release queue - // when the next command list is executed. + // after the next command list is executed. std::lock_guard<std::mutex> LockGuard(m_StaleObjectsMutex); - m_StaleVkObjects.emplace_back(m_NextCmdListNumber, new StaleVulkanObject<VulkanObjectType>(std::move(vkObject)) ); + m_StaleVkObjects.emplace_back(m_NextCmdListNumber, new StaleVulkanObject(std::move(vkObject)) ); } template void RenderDeviceVkImpl::SafeReleaseVkObject<VkBuffer> (VulkanUtilities::BufferWrapper &&Object); @@ -436,6 +442,24 @@ template void RenderDeviceVkImpl::SafeReleaseVkObject<VkSemaphore> (VulkanU template void RenderDeviceVkImpl::SafeReleaseVkObject<VkCommandPool> (VulkanUtilities::CommandPoolWrapper &&Object); +void RenderDeviceVkImpl::SafeReleaseMemoryAllocation(VulkanUtilities::VulkanMemoryAllocation&& Allocation) +{ + class StaleVulkanMemoryAllocation : public StaleVulkanObjectBase + { + public: + StaleVulkanMemoryAllocation(VulkanUtilities::VulkanMemoryAllocation &&Allocation) : + m_Allocation(std::move(Allocation)) + {} + + private: + VulkanUtilities::VulkanMemoryAllocation m_Allocation; + }; + + std::lock_guard<std::mutex> LockGuard(m_StaleObjectsMutex); + m_StaleVkObjects.emplace_back(m_NextCmdListNumber, new StaleVulkanMemoryAllocation(std::move(Allocation)) ); +} + + void RenderDeviceVkImpl::DiscardStaleVkObjects(Uint64 CmdListNumber, Uint64 FenceValue) { // Only discard these stale objects that were released before CmdListNumber diff --git a/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp index 886f2e3c..255f08fd 100644 --- a/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp +++ b/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp @@ -68,7 +68,6 @@ TextureVkImpl :: TextureVkImpl(IReferenceCounters *pRefCounters, LOG_ERROR_AND_THROW("Static Texture must be initialized with data at creation time"); const auto& LogicalDevice = pRenderDeviceVk->GetLogicalDevice(); - const auto& PhysicalDevice = pRenderDeviceVk->GetPhysicalDevice(); VkImageCreateInfo ImageCI = {}; ImageCI.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; @@ -199,41 +198,17 @@ TextureVkImpl :: TextureVkImpl(IReferenceCounters *pRefCounters, m_VulkanImage = LogicalDevice.CreateImage(ImageCI, m_Desc.Name); VkMemoryRequirements MemReqs = LogicalDevice.GetImageMemoryRequirements(m_VulkanImage); - - VkMemoryAllocateInfo MemAlloc = {}; - MemAlloc.pNext = nullptr; - MemAlloc.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; - MemAlloc.allocationSize = MemReqs.size; - + VkMemoryPropertyFlags ImageMemoryFlags = 0; if (m_Desc.Usage == USAGE_CPU_ACCESSIBLE) ImageMemoryFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; else ImageMemoryFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - // memoryTypeBits is a bitmask and contains one bit set for every supported memory type for the resource. - // Bit i is set if and only if the memory type i in the VkPhysicalDeviceMemoryProperties structure for the - // physical device is supported for the resource. - MemAlloc.memoryTypeIndex = PhysicalDevice.GetMemoryTypeIndex(MemReqs.memoryTypeBits, ImageMemoryFlags); - if (ImageMemoryFlags == VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) - { - // There must be at least one memory type with the DEVICE_LOCAL_BIT bit set - VERIFY(MemAlloc.memoryTypeIndex != VulkanUtilities::VulkanPhysicalDevice::InvalidMemoryTypeIndex, - "Vulkan spec requires that memoryTypeBits member always contains " - "at least one bit set corresponding to a VkMemoryType with a propertyFlags that has the " - "VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT bit set (11.6)"); - } - else if (MemAlloc.memoryTypeIndex != VulkanUtilities::VulkanPhysicalDevice::InvalidMemoryTypeIndex) - { - LOG_ERROR_AND_THROW("Failed to find suitable device memory type for an image"); - } - - std::string MemoryName = "Device memory for \'"; - MemoryName += m_Desc.Name; - MemoryName += '\''; - m_ImageMemory = LogicalDevice.AllocateDeviceMemory(MemAlloc, MemoryName.c_str()); - - auto err = LogicalDevice.BindImageMemory(m_VulkanImage, m_ImageMemory, 0 /*offset*/); + m_MemoryAllocation = pRenderDeviceVk->AllocateMemory(MemReqs, ImageMemoryFlags); + auto AlignedOffset = (m_MemoryAllocation.UnalignedOffset + (MemReqs.alignment-1)) & ~(MemReqs.alignment-1); + auto Memory = m_MemoryAllocation.Page->GetVkMemory(); + auto err = LogicalDevice.BindImageMemory(m_VulkanImage, Memory, AlignedOffset); CHECK_VK_ERROR_AND_THROW(err, "Failed to bind image memory"); if(bInitializeTexture) @@ -248,7 +223,11 @@ TextureVkImpl :: TextureVkImpl(IReferenceCounters *pRefCounters, if (FmtAttribs.ComponentType == COMPONENT_TYPE_DEPTH) aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT; else if (FmtAttribs.ComponentType == COMPONENT_TYPE_DEPTH_STENCIL) - aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT; + { + UNSUPPORTED("Initializing depth-stencil texture is not currently supported"); + // Only single aspect bit must be specified + aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT;// | VK_IMAGE_ASPECT_STENCIL_BIT; + } else aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; @@ -318,35 +297,16 @@ TextureVkImpl :: TextureVkImpl(IReferenceCounters *pRefCounters, VulkanUtilities::BufferWrapper StagingBuffer = LogicalDevice.CreateBuffer(VkStaginBuffCI, StagingBufferName.c_str()); VkMemoryRequirements StagingBufferMemReqs = LogicalDevice.GetBufferMemoryRequirements(StagingBuffer); - - VkMemoryAllocateInfo StagingMemAlloc = {}; - StagingMemAlloc.pNext = nullptr; - StagingMemAlloc.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; - StagingMemAlloc.allocationSize = StagingBufferMemReqs.size; - // VK_MEMORY_PROPERTY_HOST_COHERENT_BIT bit specifies that the host cache management commands vkFlushMappedMemoryRanges // and vkInvalidateMappedMemoryRanges are NOT needed to flush host writes to the device or make device writes visible // to the host (10.2) - StagingMemAlloc.memoryTypeIndex = PhysicalDevice.GetMemoryTypeIndex(StagingBufferMemReqs.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); - - VERIFY(StagingMemAlloc.memoryTypeIndex != VulkanUtilities::VulkanPhysicalDevice::InvalidMemoryTypeIndex, - "Vulkan spec requires that for a VkBuffer not created with the " - "VK_BUFFER_CREATE_SPARSE_BINDING_BIT bit set, the memoryTypeBits member always contains at least one bit set " - "corresponding to a VkMemoryType with a propertyFlags that has both the VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT bit " - "and the VK_MEMORY_PROPERTY_HOST_COHERENT_BIT bit set(11.6)"); - - std::string StagingMemoryName("Staging memory for buffer '"); - StagingMemoryName += m_Desc.Name; - StagingMemoryName += '\''; - VulkanUtilities::DeviceMemoryWrapper StagingBufferMemory = LogicalDevice.AllocateDeviceMemory(StagingMemAlloc, StagingMemoryName.c_str()); - - uint8_t *StagingData = nullptr; - err = LogicalDevice.MapMemory(StagingBufferMemory, - 0, // offset - StagingMemAlloc.allocationSize, - 0, // flags, reserved for future use - reinterpret_cast<void**>(&StagingData)); - CHECK_VK_ERROR_AND_THROW(err, "Failed to map staging memory"); + auto StagingMemoryAllocation = pRenderDeviceVk->AllocateMemory(StagingBufferMemReqs, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + auto StagingBufferMemory = StagingMemoryAllocation.Page->GetVkMemory(); + auto AlignedStagingMemOffset = (StagingMemoryAllocation.UnalignedOffset + (StagingBufferMemReqs.alignment-1)) & ~(StagingBufferMemReqs.alignment-1); + + auto *StagingData = reinterpret_cast<uint8_t*>(StagingMemoryAllocation.Page->GetCPUMemory()); + VERIFY_EXPR(StagingData != nullptr); + StagingData += AlignedStagingMemOffset; subres = 0; for(Uint32 layer = 0; layer < ImageCI.arrayLayers; ++layer) @@ -383,9 +343,7 @@ TextureVkImpl :: TextureVkImpl(IReferenceCounters *pRefCounters, } VERIFY_EXPR(subres == InitData.NumSubresources); - LogicalDevice.UnmapMemory(StagingBufferMemory); - - err = LogicalDevice.BindBufferMemory(StagingBuffer, StagingBufferMemory, 0 /*offset*/); + err = LogicalDevice.BindBufferMemory(StagingBuffer, StagingBufferMemory, AlignedStagingMemOffset); CHECK_VK_ERROR_AND_THROW(err, "Failed to bind staging bufer memory"); VulkanUtilities::CommandPoolWrapper CmdPool; @@ -436,7 +394,7 @@ TextureVkImpl :: TextureVkImpl(IReferenceCounters *pRefCounters, // until copy operation is complete. This must be done after // submitting command list for execution! pRenderDeviceVk->SafeReleaseVkObject(std::move(StagingBuffer)); - pRenderDeviceVk->SafeReleaseVkObject(std::move(StagingBufferMemory)); + pRenderDeviceVk->SafeReleaseMemoryAllocation(std::move(StagingMemoryAllocation)); } #if 0 @@ -537,7 +495,7 @@ TextureVkImpl :: ~TextureVkImpl() // Vk object can only be destroyed when it is no longer used by the GPU // Wrappers for external texture will not be destroyed as they are created with null device pointer pDeviceVkImpl->SafeReleaseVkObject(std::move(m_VulkanImage)); - pDeviceVkImpl->SafeReleaseVkObject(std::move(m_ImageMemory)); + pDeviceVkImpl->SafeReleaseMemoryAllocation(std::move(m_MemoryAllocation)); } void TextureVkImpl::UpdateData( IDeviceContext *pContext, Uint32 MipLevel, Uint32 Slice, const Box &DstBox, const TextureSubResData &SubresData ) @@ -602,7 +560,9 @@ void TextureVkImpl :: CopyData(IDeviceContext *pContext, if (FmtAttribs.ComponentType == COMPONENT_TYPE_DEPTH) aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT; else if (FmtAttribs.ComponentType == COMPONENT_TYPE_DEPTH_STENCIL) + { aspectMask = VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT; + } else aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; diff --git a/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanMemoryManager.cpp b/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanMemoryManager.cpp new file mode 100644 index 00000000..92cf3cea --- /dev/null +++ b/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanMemoryManager.cpp @@ -0,0 +1,205 @@ +/* Copyright 2015-2018 Egor Yusov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF ANY PROPRIETARY RIGHTS. + * + * In no event and under no legal theory, whether in tort (including negligence), + * contract, or otherwise, unless required by applicable law (such as deliberate + * and grossly negligent acts) or agreed to in writing, shall any Contributor be + * liable for any damages, including any direct, indirect, special, incidental, + * or consequential damages of any character arising as a result of this License or + * out of the use or inability to use the software (including but not limited to damages + * for loss of goodwill, work stoppage, computer failure or malfunction, or any and + * all other commercial damages or losses), even if such Contributor has been advised + * of the possibility of such damages. + */ + +#include "pch.h" +#include <sstream> +#include <iomanip> +#include "VulkanUtilities/VulkanMemoryManager.h" + +namespace VulkanUtilities +{ + +VulkanMemoryAllocation::~VulkanMemoryAllocation() +{ + if(Page != nullptr) + { + Page->Free(*this); + } +} + +VulkanMemoryPage::VulkanMemoryPage(VulkanMemoryManager& ParentMemoryMgr, + VkDeviceSize PageSize, + uint32_t MemoryTypeIndex, + bool MapMemory)noexcept : + m_ParentMemoryMgr(ParentMemoryMgr), + m_AllocationMgr(PageSize, ParentMemoryMgr.m_Allocator) +{ + VkMemoryAllocateInfo MemAlloc = {}; + MemAlloc.pNext = nullptr; + MemAlloc.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + MemAlloc.allocationSize = PageSize; + MemAlloc.memoryTypeIndex = MemoryTypeIndex; + + std::stringstream ss; + ss << "Device memory page. Size: " << (PageSize >> 10) << " Kb, type: " << MemoryTypeIndex; + m_VkMemory = ParentMemoryMgr.m_LogicalDevice.AllocateDeviceMemory(MemAlloc, ss.str().c_str()); + + if(MapMemory) + { + auto err = ParentMemoryMgr.m_LogicalDevice.MapMemory(m_VkMemory, + 0, // offset + PageSize, + 0, // flags, reserved for future use + &m_CPUMemory); + CHECK_VK_ERROR_AND_THROW(err, "Failed to map staging memory"); + } +} + +VulkanMemoryPage::~VulkanMemoryPage() +{ + if(m_CPUMemory != nullptr) + { + // Unmapping memory is not necessary, byt anyway + m_ParentMemoryMgr.m_LogicalDevice.UnmapMemory(m_VkMemory); + } + + VERIFY(IsEmpty(), "Destroying a page with not all allocations released"); +} + +VulkanMemoryAllocation VulkanMemoryPage::Allocate(VkDeviceSize size) +{ + std::lock_guard<std::mutex> Lock(m_Mutex); + auto Offset = m_AllocationMgr.Allocate(size); + if(Offset != Diligent::VariableSizeAllocationsManager::InvalidOffset) + { + return VulkanMemoryAllocation{this, Offset, size}; + } + else + { + return VulkanMemoryAllocation{}; + } +} + +void VulkanMemoryPage::Free(VulkanMemoryAllocation& Allocation) +{ + m_ParentMemoryMgr.OnFreeAllocation(Allocation.Size, m_CPUMemory != nullptr); + std::lock_guard<std::mutex> Lock(m_Mutex); + m_AllocationMgr.Free(Allocation.UnalignedOffset, Allocation.Size); +} + +VulkanMemoryAllocation VulkanMemoryManager::Allocate(const VkMemoryRequirements& MemReqs, VkMemoryPropertyFlags MemoryProps) +{ + // memoryTypeBits is a bitmask and contains one bit set for every supported memory type for the resource. + // Bit i is set if and only if the memory type i in the VkPhysicalDeviceMemoryProperties structure for the + // physical device is supported for the resource. + auto MemoryTypeIndex = m_PhysicalDevice.GetMemoryTypeIndex(MemReqs.memoryTypeBits, MemoryProps); + if(MemoryProps == VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) + { + // There must be at least one memory type with the DEVICE_LOCAL_BIT bit set + VERIFY(MemoryTypeIndex != VulkanUtilities::VulkanPhysicalDevice::InvalidMemoryTypeIndex, + "Vulkan spec requires that memoryTypeBits member always contains " + "at least one bit set corresponding to a VkMemoryType with a propertyFlags that has the " + "VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT bit set (11.6)"); + } + else if( (MemoryProps & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) == (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) + { + VERIFY(MemoryTypeIndex != VulkanUtilities::VulkanPhysicalDevice::InvalidMemoryTypeIndex, + "Vulkan spec requires that for a VkBuffer not created with the VK_BUFFER_CREATE_SPARSE_BINDING_BIT " + "bit set, or for a VkImage that was created with a VK_IMAGE_TILING_LINEAR value in the tiling member " + "of the VkImageCreateInfo structure passed to vkCreateImage, the memoryTypeBits member always contains " + "at least one bit set corresponding to a VkMemoryType with a propertyFlags that has both the " + "VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT bit AND the VK_MEMORY_PROPERTY_HOST_COHERENT_BIT bit set. (11.6)"); + } + else if(MemoryTypeIndex == VulkanUtilities::VulkanPhysicalDevice::InvalidMemoryTypeIndex) + { + LOG_ERROR_AND_THROW("Failed to find suitable device memory type for a buffer"); + } + + auto Size = MemReqs.size + MemReqs.alignment; + VulkanMemoryAllocation Allocation; + + std::lock_guard<std::mutex> Lock(m_Mutex); + auto range = m_Pages.equal_range(MemoryTypeIndex); + for(auto page_it = range.first; page_it != range.second; ++page_it) + { + Allocation = page_it->second.Allocate(Size); + if(Allocation.Page != nullptr) + break; + } + + bool HostVisible = (MemoryProps & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0; + size_t stat_ind = HostVisible ? 1 : 0; + if(Allocation.Page == nullptr) + { + auto PageSize = HostVisible ? m_HostVisiblePageSize : m_DeviceLocalPageSize; + while(PageSize < Size) + PageSize *= 2; + + m_CurrAllocatedSize[stat_ind] += PageSize; + m_PeakAllocatedSize[stat_ind] = std::max(m_PeakAllocatedSize[stat_ind], m_CurrAllocatedSize[stat_ind]); + + auto it = m_Pages.emplace(MemoryTypeIndex, VulkanMemoryPage{*this, PageSize, MemoryTypeIndex, HostVisible}); + LOG_INFO_MESSAGE("VulkanMemoryManager '", m_MgrName, "': created new ", (HostVisible ? "host-visible" : "device-local"), " page. Size: ", + std::fixed, std::setprecision(2), PageSize / double{1 << 20}, " MB. Memory type: ", MemoryTypeIndex, + ". Current allocated size: ", std::fixed, std::setprecision(2), m_CurrAllocatedSize[stat_ind] / double{1 << 20}); + Allocation = it->second.Allocate(Size); + VERIFY(Allocation.Page != nullptr, "Failed to allocate new memory page"); + } + + m_CurrUsedSize[stat_ind].fetch_add(Size); + m_PeakUsedSize[stat_ind] = std::max(m_PeakUsedSize[stat_ind], static_cast<VkDeviceSize>(m_CurrUsedSize[stat_ind].load())); + + return Allocation; +} + +void VulkanMemoryManager::ShrinkMemory() +{ + std::lock_guard<std::mutex> Lock(m_Mutex); + auto it = m_Pages.begin(); + while(it != m_Pages.end()) + { + auto curr_it = it; + ++it; + auto& Page = curr_it->second; + if(Page.IsEmpty()) + { + auto PageSize = Page.GetPageSize(); + bool IsHostVisible = Page.GetCPUMemory() != nullptr; + m_CurrAllocatedSize[IsHostVisible ? 1 : 0] -= PageSize; + LOG_INFO_MESSAGE("VulkanMemoryManager '", m_MgrName, "': destroying ", (IsHostVisible ? "host-visible" : "device-local"), " page. Size: ", + std::fixed, std::setprecision(2), PageSize / double{1 << 20}, + ". Current allocated size: ", std::fixed, std::setprecision(2), m_CurrAllocatedSize[IsHostVisible ? 1 : 0] / double{1 << 20}); + m_Pages.erase(curr_it); + } + } +} + +void VulkanMemoryManager::OnFreeAllocation(VkDeviceSize Size, bool IsHostVisble) +{ + m_CurrUsedSize[IsHostVisble ? 1 : 0].fetch_add( -static_cast<int64_t>(Size) ); +} + +VulkanMemoryManager::~VulkanMemoryManager() +{ + LOG_INFO_MESSAGE("VulkanMemoryManager '", m_MgrName, "' stats:\n" + " Peak used/peak allocated device-local memory size: ", + std::fixed, std::setprecision(2), m_PeakUsedSize[0] / double{1 << 20}, "/", + std::fixed, std::setprecision(2), m_PeakAllocatedSize[0] / double{1 << 20}, " MB. " + "\n Peak used/peak allocated host-visible memory size: ", + std::fixed, std::setprecision(2), m_PeakUsedSize[1] / double{1 << 20}, "/", + std::fixed, std::setprecision(2), m_PeakAllocatedSize[1] / double{1 << 20}, " MB."); + VERIFY(m_Pages.empty(), "Not all pages have been released"); + VERIFY_EXPR(m_CurrAllocatedSize[0] == 0 && m_CurrAllocatedSize[1] == 0 && m_CurrUsedSize[0] == 0 && m_CurrUsedSize[1] == 0); +} + +} |
