diff options
| author | Egor Yusov <egor.yusov@gmail.com> | 2018-09-09 03:13:04 +0000 |
|---|---|---|
| committer | Egor Yusov <egor.yusov@gmail.com> | 2018-09-09 03:13:04 +0000 |
| commit | d7dda5023e6dfa999286a45ed314158330c18a5f (patch) | |
| tree | f01498bdad1032243bd7a97732eb9284152c6d60 /Graphics/GraphicsEngineVulkan | |
| parent | Fixed debug check in ShaderResourceBindingD3D12Impl::InitializeStaticResources() (diff) | |
| download | DiligentCore-d7dda5023e6dfa999286a45ed314158330c18a5f.tar.gz DiligentCore-d7dda5023e6dfa999286a45ed314158330c18a5f.zip | |
Added alignment parameter to allocation managers (fixed https://github.com/DiligentGraphics/DiligentCore/issues/29)
Diffstat (limited to 'Graphics/GraphicsEngineVulkan')
13 files changed, 201 insertions, 258 deletions
diff --git a/Graphics/GraphicsEngineVulkan/include/BufferVkImpl.h b/Graphics/GraphicsEngineVulkan/include/BufferVkImpl.h index 72b5e217..397a4198 100644 --- a/Graphics/GraphicsEngineVulkan/include/BufferVkImpl.h +++ b/Graphics/GraphicsEngineVulkan/include/BufferVkImpl.h @@ -87,7 +87,7 @@ public: DvpVerifyDynamicAllocation(pCtx); #endif auto& DynAlloc = m_DynamicAllocations[CtxId]; - return static_cast<Uint32>(DynAlloc.Offset); + return static_cast<Uint32>(DynAlloc.AlignedOffset); } } VkBuffer GetVkBuffer()const override final; @@ -117,7 +117,8 @@ private: virtual void CreateViewInternal( const struct BufferViewDesc& ViewDesc, IBufferView** ppView, bool bIsDefaultView )override; VulkanUtilities::BufferViewWrapper CreateView(struct BufferViewDesc &ViewDesc); - VkAccessFlags m_AccessFlags = 0; + VkAccessFlags m_AccessFlags = 0; + Uint32 m_DynamicOffsetAlignment = 0; #ifdef DEVELOPMENT std::vector< std::pair<MAP_TYPE, Uint32> > m_DvpMapType; diff --git a/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h b/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h index 050e0103..06a6a744 100644 --- a/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h +++ b/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h @@ -172,7 +172,7 @@ public: return m_DynamicDescriptorPool.Allocate(SetLayout); } - VulkanDynamicAllocation AllocateDynamicSpace(Uint32 SizeInBytes); + VulkanDynamicAllocation AllocateDynamicSpace(Uint32 SizeInBytes, Uint32 Alignment); void ResetRenderTargets(); Int64 GetContextFrameNumber()const{return m_ContextFrameNumber;} @@ -191,11 +191,12 @@ private: struct BufferToTextureCopyInfo { - Uint32 RowSize = 0; - Uint32 Stride = 0; - Uint32 DepthStride = 0; - Uint32 MemorySize = 0; - Uint32 RowCount = 0; + Uint32 RowSize = 0; + Uint32 Stride = 0; + Uint32 StrideInTexels = 0; + Uint32 DepthStride = 0; + Uint32 MemorySize = 0; + Uint32 RowCount = 0; Box Region; }; BufferToTextureCopyInfo GetBufferToTextureCopyInfo(const TextureDesc& TexDesc, @@ -204,6 +205,7 @@ private: void CopyBufferToTexture(VkBuffer vkBuffer, Uint32 BufferOffset, + Uint32 BufferRowStrideInTexels, const Box& Region, TextureVkImpl& TextureVk, Uint32 MipLevel, diff --git a/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h b/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h index 227526b2..49db82b9 100644 --- a/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h +++ b/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h @@ -125,10 +125,10 @@ public: } std::shared_ptr<const VulkanUtilities::VulkanInstance> GetVulkanInstance()const{return m_VulkanInstance;} - const VulkanUtilities::VulkanPhysicalDevice& GetPhysicalDevice(){return *m_PhysicalDevice;} - const VulkanUtilities::VulkanLogicalDevice& GetLogicalDevice() {return *m_LogicalVkDevice;} + const VulkanUtilities::VulkanPhysicalDevice& GetPhysicalDevice()const {return *m_PhysicalDevice;} + const VulkanUtilities::VulkanLogicalDevice& GetLogicalDevice () {return *m_LogicalVkDevice;} FramebufferCache& GetFramebufferCache(){return m_FramebufferCache;} - RenderPassCache& GetRenderPassCache(){return m_RenderPassCache;} + RenderPassCache& GetRenderPassCache() {return m_RenderPassCache;} VulkanUtilities::VulkanMemoryAllocation AllocateMemory(const VkMemoryRequirements& MemReqs, VkMemoryPropertyFlags MemoryProperties) { diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanDynamicHeap.h b/Graphics/GraphicsEngineVulkan/include/VulkanDynamicHeap.h index 3e3d0c7f..79a95abe 100644 --- a/Graphics/GraphicsEngineVulkan/include/VulkanDynamicHeap.h +++ b/Graphics/GraphicsEngineVulkan/include/VulkanDynamicHeap.h @@ -25,10 +25,10 @@ #include <mutex> #include "Vulkan.h" -#include "RingBuffer.h" #include "VulkanUtilities/VulkanMemoryManager.h" #include "VulkanUtilities/VulkanLogicalDevice.h" #include "VulkanUtilities/VulkanObjectWrappers.h" +#include "DynamicHeap.h" namespace Diligent { @@ -47,9 +47,9 @@ struct VulkanDynamicAllocation { VulkanDynamicAllocation()noexcept{} - VulkanDynamicAllocation(VulkanDynamicMemoryManager& _DynamicMemMgr, size_t _Offset, size_t _Size)noexcept : + VulkanDynamicAllocation(VulkanDynamicMemoryManager& _DynamicMemMgr, size_t _AlignedOffset, size_t _Size)noexcept : pDynamicMemMgr(&_DynamicMemMgr), - Offset (_Offset), + AlignedOffset (_AlignedOffset), Size (_Size) {} @@ -57,15 +57,15 @@ struct VulkanDynamicAllocation VulkanDynamicAllocation& operator = (const VulkanDynamicAllocation&) = delete; VulkanDynamicAllocation (VulkanDynamicAllocation&& rhs)noexcept : pDynamicMemMgr(rhs.pDynamicMemMgr), - Offset (rhs.Offset), + AlignedOffset (rhs.AlignedOffset), Size (rhs.Size) #ifdef DEVELOPMENT , dvpFrameNumber(rhs.dvpFrameNumber) #endif { rhs.pDynamicMemMgr = nullptr; - rhs.Offset = 0; - rhs.Size = 0; + rhs.AlignedOffset = 0; + rhs.Size = 0; #ifdef DEVELOPMENT rhs.dvpFrameNumber = 0; #endif @@ -74,20 +74,20 @@ struct VulkanDynamicAllocation VulkanDynamicAllocation& operator = (VulkanDynamicAllocation&& rhs)noexcept // Must be noexcept on MSVC, so can't use = default { pDynamicMemMgr = rhs.pDynamicMemMgr; - Offset = rhs.Offset; + AlignedOffset = rhs.AlignedOffset; Size = rhs.Size; rhs.pDynamicMemMgr = nullptr; - rhs.Offset = 0; + rhs.AlignedOffset = 0; rhs.Size = 0; #ifdef DEVELOPMENT - dvpFrameNumber = rhs.dvpFrameNumber; + dvpFrameNumber = rhs.dvpFrameNumber; rhs.dvpFrameNumber = 0; #endif return *this; } VulkanDynamicMemoryManager* pDynamicMemMgr = nullptr; - size_t Offset = 0; // Offset from the start of the buffer resource + size_t AlignedOffset = 0; // Offset from the start of the buffer size_t Size = 0; // Reserved size of this allocation #ifdef DEVELOPMENT Int64 dvpFrameNumber = 0; @@ -95,159 +95,36 @@ struct VulkanDynamicAllocation }; -// Having global ring buffer shared between all contexts is inconvinient because all contexts -// must share the same frame. Having individual ring bufer per context may result in a lot of unused -// memory. As a result, ring buffer is not currently used for dynamic memory management. -// Instead, every dynamic heap allocates pages from the global dynamic memory manager. -class RingBufferAllocationStrategy -{ -public: - using OffsetType = RingBuffer::OffsetType; - static constexpr const OffsetType InvalidOffset = RingBuffer::InvalidOffset; - - RingBufferAllocationStrategy(IMemoryAllocator& Allocator, - Uint32 Size) : - m_RingBuffer(Size, Allocator) - {} - - RingBufferAllocationStrategy (const RingBufferAllocationStrategy&) = delete; - RingBufferAllocationStrategy (RingBufferAllocationStrategy&&) = delete; - RingBufferAllocationStrategy& operator= (const RingBufferAllocationStrategy&) = delete; - RingBufferAllocationStrategy& operator= (RingBufferAllocationStrategy&&) = delete; - - void DiscardAllocations(const std::vector<std::pair<OffsetType, OffsetType>>& Allocations, Uint64 FenceValue) - { - std::lock_guard<std::mutex> Lock(m_RingBufferMtx); - m_RingBuffer.FinishCurrentFrame(FenceValue); - } - - void ReleaseStaleAllocations(Uint64 LastCompletedFenceValue) - { - std::lock_guard<std::mutex> Lock(m_RingBufferMtx); - m_RingBuffer.ReleaseCompletedFrames(LastCompletedFenceValue); - } - - OffsetType Allocate(OffsetType SizeInBytes) - { - std::lock_guard<std::mutex> Lock(m_RingBufferMtx); - return m_RingBuffer.Allocate(SizeInBytes); - } - - OffsetType GetSize() const{return m_RingBuffer.GetMaxSize();} - OffsetType GetUsedSize()const{return m_RingBuffer.GetUsedSize();} -private: - std::mutex m_RingBufferMtx; - RingBuffer m_RingBuffer; -}; - - -class ListBasedAllocationStrategy -{ -public: - using OffsetType = VariableSizeAllocationsManager::OffsetType; - static constexpr const OffsetType InvalidOffset = VariableSizeAllocationsManager::InvalidOffset; - - ListBasedAllocationStrategy(IMemoryAllocator& Allocator, - Uint32 Size) : - m_AllocationsMgr(Size, Allocator) - {} - - ListBasedAllocationStrategy (const ListBasedAllocationStrategy&) = delete; - ListBasedAllocationStrategy (ListBasedAllocationStrategy&&) = delete; - ListBasedAllocationStrategy& operator= (const ListBasedAllocationStrategy&) = delete; - ListBasedAllocationStrategy& operator= (ListBasedAllocationStrategy&&) = delete; - - void DiscardAllocations(const std::vector<std::pair<OffsetType, OffsetType>>& Allocations, Uint64 FenceValue) - { - std::lock_guard<std::mutex> Lock(m_ReleaseQueueMtx); - for(const auto& Allocation : Allocations) - m_ReleaseQueue.emplace_back(Allocation.first, Allocation.second, FenceValue); - } - - void ReleaseStaleAllocations(Uint64 LastCompletedFenceValue) - { - std::lock_guard<std::mutex> MgrLock(m_AllocationsMgrMtx); - std::lock_guard<std::mutex> QueueLock(m_ReleaseQueueMtx); - while (!m_ReleaseQueue.empty()) - { - auto &FirstAllocation = m_ReleaseQueue.front(); - if (FirstAllocation.FenceValue <= LastCompletedFenceValue) - { - m_AllocationsMgr.Free(FirstAllocation.Offset, FirstAllocation.Size); - m_ReleaseQueue.pop_front(); - } - else - break; - } - } - - OffsetType Allocate(OffsetType SizeInBytes) - { - std::lock_guard<std::mutex> Lock(m_AllocationsMgrMtx); - return m_AllocationsMgr.Allocate(SizeInBytes); - } - OffsetType GetSize() const{return m_AllocationsMgr.GetMaxSize();} - OffsetType GetUsedSize()const{return m_AllocationsMgr.GetUsedSize();} - -private: - std::mutex m_AllocationsMgrMtx; - VariableSizeAllocationsManager m_AllocationsMgr; - - struct StaleAllocationInfo - { - const OffsetType Offset; - const OffsetType Size; - const Uint64 FenceValue; - StaleAllocationInfo(OffsetType _Offset, - OffsetType _Size, - Uint64 _FenceValue) : - Offset (_Offset), - Size (_Size), - FenceValue(_FenceValue) - {} - }; - std::deque< StaleAllocationInfo > m_ReleaseQueue; - std::mutex m_ReleaseQueueMtx; -}; // We cannot use global memory manager for dynamic resources because they // need to use the same Vulkan buffer -class VulkanDynamicMemoryManager +class VulkanDynamicMemoryManager : public DynamicHeap::MasterBlockListBasedManager // or MasterBlockRingBufferBasedManager { public: - using AllocationStategy = ListBasedAllocationStrategy; // or RingBufferAllocationStrategy - using OffsetType = AllocationStategy::OffsetType; + using TBase = DynamicHeap::MasterBlockListBasedManager; + using OffsetType = TBase::OffsetType; + using TBase::InvalidOffset; + using MasterBlock = TBase::MasterBlock; VulkanDynamicMemoryManager(IMemoryAllocator& Allocator, class RenderDeviceVkImpl& DeviceVk, Uint32 Size); ~VulkanDynamicMemoryManager(); - VulkanDynamicMemoryManager (const VulkanDynamicMemoryManager&) = delete; - VulkanDynamicMemoryManager (VulkanDynamicMemoryManager&&) = delete; - VulkanDynamicMemoryManager& operator= (const VulkanDynamicMemoryManager&) = delete; - VulkanDynamicMemoryManager& operator= (VulkanDynamicMemoryManager&&) = delete; - - void ReleaseStaleAllocations(Uint64 LastCompletedFenceValue) - { - m_AllocationStrategy.ReleaseStaleAllocations(LastCompletedFenceValue); - } - - void DiscardAllocations(const std::vector<std::pair<OffsetType, OffsetType>>& Allocations, Uint64 FenceValue) - { - m_AllocationStrategy.DiscardAllocations(Allocations, FenceValue); - } + VulkanDynamicMemoryManager (const VulkanDynamicMemoryManager&) = delete; + VulkanDynamicMemoryManager ( VulkanDynamicMemoryManager&&) = delete; + VulkanDynamicMemoryManager& operator= (const VulkanDynamicMemoryManager&) = delete; + VulkanDynamicMemoryManager& operator= ( VulkanDynamicMemoryManager&&) = delete; VkBuffer GetVkBuffer() const{return m_VkBuffer;} Uint8* GetCPUAddress()const{return m_CPUAddress;} void Destroy(); - static constexpr const Uint32 MinAlignment = 1024; + static constexpr const Uint32 MasterBlockAlignment = 1024; + MasterBlock AllocateMasterBlock(OffsetType SizeInBytes, OffsetType Alignment); private: - friend class VulkanDynamicHeap; - OffsetType Allocate(OffsetType SizeInBytes); RenderDeviceVkImpl& m_DeviceVk; VulkanUtilities::BufferWrapper m_VkBuffer; @@ -255,8 +132,6 @@ private: Uint8* m_CPUAddress; const VkDeviceSize m_DefaultAlignment; - AllocationStategy m_AllocationStrategy; - OffsetType m_TotalPeakSize = 0; }; @@ -267,7 +142,7 @@ public: VulkanDynamicHeap(VulkanDynamicMemoryManager& DynamicMemMgr, std::string HeapName, Uint32 PageSize) : m_DynamicMemMgr(DynamicMemMgr), m_HeapName(std::move(HeapName)), - m_PagSize(PageSize) + m_MasterBlockSize(PageSize) {} VulkanDynamicHeap (const VulkanDynamicHeap&) = delete; @@ -280,17 +155,18 @@ public: VulkanDynamicAllocation Allocate(Uint32 SizeInBytes, Uint32 Alignment); void FinishFrame(Uint64 FenceValue); - using OffsetType = VulkanDynamicMemoryManager::AllocationStategy::OffsetType; - static constexpr OffsetType InvalidOffset = VulkanDynamicMemoryManager::AllocationStategy::InvalidOffset; + using OffsetType = VulkanDynamicMemoryManager::OffsetType; + using MasterBlock = VulkanDynamicMemoryManager::MasterBlock; + static constexpr OffsetType InvalidOffset = VulkanDynamicMemoryManager::InvalidOffset; private: VulkanDynamicMemoryManager& m_DynamicMemMgr; const std::string m_HeapName; - std::vector<std::pair<OffsetType, OffsetType>> m_Allocations; + std::vector<MasterBlock> m_MasterBlocks; OffsetType m_CurrOffset = InvalidOffset; - const Uint32 m_PagSize; + const Uint32 m_MasterBlockSize; Uint32 m_AvailableSize = 0; Uint32 m_CurrAllocatedSize = 0; diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanMemoryManager.h b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanMemoryManager.h index 270a3030..bb0f4f59 100644 --- a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanMemoryManager.h +++ b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanMemoryManager.h @@ -112,7 +112,7 @@ public: VkDeviceSize GetPageSize()const{return m_AllocationMgr.GetMaxSize();} VkDeviceSize GetUsedSize()const{return m_AllocationMgr.GetUsedSize();} - VulkanMemoryAllocation Allocate(VkDeviceSize size); + VulkanMemoryAllocation Allocate(VkDeviceSize size, VkDeviceSize alignment); VkDeviceMemory GetVkMemory()const{return m_VkMemory;} void* GetCPUMemory()const{return m_CPUMemory;} @@ -121,7 +121,7 @@ private: friend struct VulkanMemoryAllocation; // Memory is reclaimed immediately. The application is responsible to ensure it is not in use by the GPU - void Free(VulkanMemoryAllocation& Allocation); + void Free(VulkanMemoryAllocation&& Allocation); VulkanMemoryManager& m_ParentMemoryMgr; std::mutex m_Mutex; diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanPhysicalDevice.h b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanPhysicalDevice.h index 5e67138e..86dee23c 100644 --- a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanPhysicalDevice.h +++ b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanPhysicalDevice.h @@ -39,14 +39,14 @@ namespace VulkanUtilities static std::unique_ptr<VulkanPhysicalDevice> Create(VkPhysicalDevice vkDevice); - uint32_t FindQueueFamily(VkQueueFlags QueueFlags)const; - VkPhysicalDevice GetVkDeviceHandle()const{return m_VkDevice;} - bool IsExtensionSupported(const char* ExtensionName)const; - bool CheckPresentSupport(uint32_t queueFamilyIndex, VkSurfaceKHR VkSurface)const; + uint32_t FindQueueFamily (VkQueueFlags QueueFlags) const; + VkPhysicalDevice GetVkDeviceHandle () const { return m_VkDevice; } + bool IsExtensionSupported(const char* ExtensionName) const; + bool CheckPresentSupport (uint32_t queueFamilyIndex, VkSurfaceKHR VkSurface) const; static constexpr uint32_t InvalidMemoryTypeIndex = static_cast<uint32_t>(-1); uint32_t GetMemoryTypeIndex(uint32_t typeBits, VkMemoryPropertyFlags properties)const; - const VkPhysicalDeviceProperties& GetProperties()const{return m_Properties;} + const VkPhysicalDeviceProperties& GetProperties() const {return m_Properties;} private: VulkanPhysicalDevice(VkPhysicalDevice vkDevice); diff --git a/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp index ce0e8f4b..ba9c2ca6 100644 --- a/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp +++ b/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp @@ -69,6 +69,8 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters* pRefCounters, } const auto& LogicalDevice = pRenderDeviceVk->GetLogicalDevice(); + const auto& DeviceLimits = pRenderDeviceVk->GetPhysicalDevice().GetProperties().limits; + m_DynamicOffsetAlignment = std::max(Uint32{4}, static_cast<Uint32>(DeviceLimits.optimalBufferCopyOffsetAlignment)); VkBufferCreateInfo VkBuffCI = {}; VkBuffCI.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; @@ -95,6 +97,12 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters* pRefCounters, // // So we have to set both VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT and VK_BUFFER_USAGE_STORAGE_BUFFER_BIT bits VkBuffCI.usage |= VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + + // Each element of pDynamicOffsets of vkCmdBindDescriptorSets function which corresponds to a descriptor + // binding with type VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC must be a multiple of + // VkPhysicalDeviceLimits::minStorageBufferOffsetAlignment (13.2.5) + m_DynamicOffsetAlignment = std::max(m_DynamicOffsetAlignment, static_cast<Uint32>(DeviceLimits.minTexelBufferOffsetAlignment)); + m_DynamicOffsetAlignment = std::max(m_DynamicOffsetAlignment, static_cast<Uint32>(DeviceLimits.minStorageBufferOffsetAlignment)); } if (m_Desc.BindFlags & BIND_SHADER_RESOURCE) { @@ -102,6 +110,9 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters* pRefCounters, // HLSL buffer SRV are mapped to storge buffers in GLSL, so we need to set both // VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER and VK_BUFFER_USAGE_STORAGE_BUFFER_BIT flags VkBuffCI.usage |= VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + + m_DynamicOffsetAlignment = std::max(m_DynamicOffsetAlignment, static_cast<Uint32>(DeviceLimits.minTexelBufferOffsetAlignment)); + m_DynamicOffsetAlignment = std::max(m_DynamicOffsetAlignment, static_cast<Uint32>(DeviceLimits.minStorageBufferOffsetAlignment)); } if (m_Desc.BindFlags & BIND_VERTEX_BUFFER) VkBuffCI.usage |= VK_BUFFER_USAGE_VERTEX_BUFFER_BIT; @@ -110,8 +121,15 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters* pRefCounters, if (m_Desc.BindFlags & BIND_INDIRECT_DRAW_ARGS) VkBuffCI.usage |= VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT; if (m_Desc.BindFlags & BIND_UNIFORM_BUFFER) + { VkBuffCI.usage |= VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT; + // Each element of pDynamicOffsets parameter of vkCmdBindDescriptorSets function which corresponds to a descriptor + // binding with type VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC must be a multiple of + // VkPhysicalDeviceLimits::minUniformBufferOffsetAlignment (13.2.5) + m_DynamicOffsetAlignment = std::max(m_DynamicOffsetAlignment, static_cast<Uint32>(DeviceLimits.minUniformBufferOffsetAlignment)); + } + if(m_Desc.Usage == USAGE_DYNAMIC) { auto CtxCount = 1 + pRenderDeviceVk->GetNumDeferredContexts(); @@ -148,10 +166,11 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters* pRefCounters, else BufferMemoryFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + VERIFY( IsPowerOfTwo(MemReqs.alignment), "Alignment is not power of 2!"); m_MemoryAllocation = pRenderDeviceVk->AllocateMemory(MemReqs, BufferMemoryFlags); - VERIFY( (MemReqs.alignment & (MemReqs.alignment-1)) == 0, "Alignment is not power of 2!"); - auto AlignedOffset = (m_MemoryAllocation.UnalignedOffset + (MemReqs.alignment-1)) & ~(MemReqs.alignment-1); + auto AlignedOffset = Align(m_MemoryAllocation.UnalignedOffset, MemReqs.alignment); + VERIFY(m_MemoryAllocation.Size >= MemReqs.size + (AlignedOffset - m_MemoryAllocation.UnalignedOffset), "Size of memory allocation is too small"); auto Memory = m_MemoryAllocation.Page->GetVkMemory(); auto err = LogicalDevice.BindBufferMemory(m_VulkanBuffer, Memory, AlignedOffset); CHECK_VK_ERROR_AND_THROW(err, "Failed to bind buffer memory"); @@ -168,15 +187,17 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters* pRefCounters, VulkanUtilities::BufferWrapper StagingBuffer = LogicalDevice.CreateBuffer(VkStaginBuffCI, StagingBufferName.c_str()); VkMemoryRequirements StagingBufferMemReqs = LogicalDevice.GetBufferMemoryRequirements(StagingBuffer); + VERIFY( IsPowerOfTwo(StagingBufferMemReqs.alignment), "Alignment is not power of 2!"); // VK_MEMORY_PROPERTY_HOST_COHERENT_BIT bit specifies that the host cache management commands vkFlushMappedMemoryRanges // and vkInvalidateMappedMemoryRanges are NOT needed to flush host writes to the device or make device writes visible // to the host (10.2) auto StagingMemoryAllocation = pRenderDeviceVk->AllocateMemory(StagingBufferMemReqs, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); auto StagingBufferMemory = StagingMemoryAllocation.Page->GetVkMemory(); - auto AlignedStagingMemOffset = (StagingMemoryAllocation.UnalignedOffset + (StagingBufferMemReqs.alignment-1)) & ~(StagingBufferMemReqs.alignment-1); + auto AlignedStagingMemOffset = Align(StagingMemoryAllocation.UnalignedOffset, StagingBufferMemReqs.alignment); + VERIFY_EXPR(StagingMemoryAllocation.Size >= StagingBufferMemReqs.size + (AlignedStagingMemOffset - StagingMemoryAllocation.UnalignedOffset)); - auto *StagingData = reinterpret_cast<uint8_t*>(StagingMemoryAllocation.Page->GetCPUMemory()); + auto* StagingData = reinterpret_cast<uint8_t*>(StagingMemoryAllocation.Page->GetCPUMemory()); if (StagingData == nullptr) LOG_BUFFER_ERROR_AND_THROW("Failed to allocate staging data"); memcpy(StagingData + AlignedStagingMemOffset, BuffData.pData, BuffData.DataSize); @@ -334,7 +355,7 @@ void BufferVkImpl :: Map(IDeviceContext* pContext, MAP_TYPE MapType, Uint32 MapF auto& DynAllocation = m_DynamicAllocations[pDeviceContextVk->GetContextId()]; if ( (MapFlags & MAP_FLAG_DISCARD) != 0 || DynAllocation.pDynamicMemMgr == nullptr ) { - DynAllocation = pDeviceContextVk->AllocateDynamicSpace(m_Desc.uiSizeInBytes); + DynAllocation = pDeviceContextVk->AllocateDynamicSpace(m_Desc.uiSizeInBytes, m_DynamicOffsetAlignment); } else { @@ -345,7 +366,7 @@ void BufferVkImpl :: Map(IDeviceContext* pContext, MAP_TYPE MapType, Uint32 MapF if (DynAllocation.pDynamicMemMgr != nullptr) { auto* CPUAddress = DynAllocation.pDynamicMemMgr->GetCPUAddress(); - pMappedData = CPUAddress + DynAllocation.Offset; + pMappedData = CPUAddress + DynAllocation.AlignedOffset; } else { @@ -415,11 +436,11 @@ void BufferVkImpl::Unmap( IDeviceContext* pContext, MAP_TYPE MapType, Uint32 Map else if (m_Desc.Usage == USAGE_DYNAMIC) { VERIFY( MapFlags & (MAP_FLAG_DISCARD | MAP_FLAG_DO_NOT_SYNCHRONIZE), "Vk buffer must be mapped for writing with MAP_FLAG_DISCARD or MAP_FLAG_DO_NOT_SYNCHRONIZE flag"); - if(m_VulkanBuffer != VK_NULL_HANDLE) + if (m_VulkanBuffer != VK_NULL_HANDLE) { - auto &DynAlloc = m_DynamicAllocations[CtxId]; + auto& DynAlloc = m_DynamicAllocations[CtxId]; auto vkSrcBuff = DynAlloc.pDynamicMemMgr->GetVkBuffer(); - pDeviceContextVk->UpdateBufferRegion(this, 0, m_Desc.uiSizeInBytes, vkSrcBuff, DynAlloc.Offset); + pDeviceContextVk->UpdateBufferRegion(this, 0, m_Desc.uiSizeInBytes, vkSrcBuff, DynAlloc.AlignedOffset); } } } diff --git a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp index 1a0a3d41..9be700a1 100644 --- a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp +++ b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp @@ -1074,7 +1074,9 @@ namespace Diligent #endif VERIFY_EXPR( static_cast<size_t>(NumBytes) == NumBytes ); - auto TmpSpace = m_UploadHeap.Allocate(static_cast<size_t>(NumBytes), 0); + constexpr size_t Alignment = 4; + // Source buffer offset must be multiple of 4 (18.4) + auto TmpSpace = m_UploadHeap.Allocate(static_cast<size_t>(NumBytes), Alignment); memcpy(TmpSpace.CPUAddress, pData, static_cast<size_t>(NumBytes)); UpdateBufferRegion(pBuffVk, DstOffset, NumBytes, TmpSpace.vkBuffer, TmpSpace.AlignedOffset); // The allocation will stay in the upload heap until the end of the frame at which point all upload @@ -1161,7 +1163,19 @@ namespace Diligent CopyInfo.RowCount = UpdateRegionHeight; } - CopyInfo.Stride = CopyInfo.RowSize; + const auto& DeviceLimits = m_pDevice.RawPtr<const RenderDeviceVkImpl>()->GetPhysicalDevice().GetProperties().limits; + CopyInfo.Stride = Align(CopyInfo.RowSize, static_cast<Uint32>(DeviceLimits.optimalBufferCopyRowPitchAlignment)); + if (FmtAttribs.ComponentType == COMPONENT_TYPE_COMPRESSED) + { + // If the calling command’s VkImage parameter is a compressed image, + // bufferRowLength must be a multiple of the compressed texel block width + // In texels (not even in compressed blocks!) + CopyInfo.StrideInTexels = CopyInfo.Stride / Uint32{FmtAttribs.ComponentSize} * Uint32{FmtAttribs.BlockWidth}; + } + else + { + CopyInfo.StrideInTexels = CopyInfo.Stride / (Uint32{FmtAttribs.ComponentSize} * Uint32{FmtAttribs.NumComponents}); + } CopyInfo.DepthStride = CopyInfo.RowCount * CopyInfo.Stride; CopyInfo.MemorySize = UpdateRegionDepth * CopyInfo.DepthStride; CopyInfo.Region = Region; @@ -1183,7 +1197,9 @@ namespace Diligent const auto UpdateRegionDepth = CopyInfo.Region.MaxZ - CopyInfo.Region.MinZ; // For UpdateTextureRegion(), use UploadHeap, not dynamic heap - static constexpr const size_t BufferOffsetAlignment = 4; // bufferOffset of VkBufferImageCopy must be a multiple of 4 (18.4) + const auto& DeviceLimits = m_pDevice.RawPtr<const RenderDeviceVkImpl>()->GetPhysicalDevice().GetProperties().limits; + // Source buffer offset must be multiple of 4 (18.4) + auto BufferOffsetAlignment = std::max(DeviceLimits.optimalBufferCopyOffsetAlignment, VkDeviceSize{4}); auto Allocation = m_UploadHeap.Allocate(CopyInfo.MemorySize, BufferOffsetAlignment); // The allocation will stay in the upload heap until the end of the frame at which point all upload // pages will be discarded @@ -1212,6 +1228,7 @@ namespace Diligent } CopyBufferToTexture(Allocation.vkBuffer, static_cast<Uint32>(Allocation.AlignedOffset), + CopyInfo.StrideInTexels, CopyInfo.Region, TextureVk, MipLevel, @@ -1220,6 +1237,7 @@ namespace Diligent void DeviceContextVkImpl::CopyBufferToTexture(VkBuffer vkBuffer, Uint32 BufferOffset, + Uint32 BufferRowStrideInTexels, const Box& Region, TextureVkImpl& TextureVk, Uint32 MipLevel, @@ -1238,7 +1256,7 @@ namespace Diligent // bufferRowLength and bufferImageHeight specify the data in buffer memory as a subregion of a larger two- or // three-dimensional image, and control the addressing calculations of data in buffer memory. If either of these // values is zero, that aspect of the buffer memory is considered to be tightly packed according to the imageExtent (18.4). - CopyRegion.bufferRowLength = 0; + CopyRegion.bufferRowLength = BufferRowStrideInTexels; CopyRegion.bufferImageHeight = 0; const auto& TexDesc = TextureVk.GetDesc(); @@ -1300,10 +1318,12 @@ namespace Diligent { const auto& TexDesc = TextureVk.GetDesc(); auto CopyInfo = GetBufferToTextureCopyInfo(TexDesc, MipLevel, MapRegion); - auto Allocation = AllocateDynamicSpace(CopyInfo.MemorySize); - VERIFY( (Allocation.Offset % 4) == 0, "Allocation offset must be at least 32-bit algined"); + const auto& DeviceLimits = m_pDevice.RawPtr<RenderDeviceVkImpl>()->GetPhysicalDevice().GetProperties().limits; + // Source buffer offset must be multiple of 4 (18.4) + auto Alignment = std::max(DeviceLimits.optimalBufferCopyOffsetAlignment, VkDeviceSize{4}); + auto Allocation = AllocateDynamicSpace(CopyInfo.MemorySize, static_cast<Uint32>(Alignment)); - MappedData.pData = reinterpret_cast<Uint8*>(Allocation.pDynamicMemMgr->GetCPUAddress()) + Allocation.Offset; + MappedData.pData = reinterpret_cast<Uint8*>(Allocation.pDynamicMemMgr->GetCPUAddress()) + Allocation.AlignedOffset; MappedData.Stride = CopyInfo.Stride; MappedData.DepthStride = CopyInfo.DepthStride; @@ -1322,7 +1342,8 @@ namespace Diligent { auto& MappedTex = UploadSpaceIt->second; CopyBufferToTexture(MappedTex.Allocation.pDynamicMemMgr->GetVkBuffer(), - static_cast<Uint32>(MappedTex.Allocation.Offset), + static_cast<Uint32>(MappedTex.Allocation.AlignedOffset), + MappedTex.CopyInfo.StrideInTexels, MappedTex.CopyInfo.Region, TextureVk, MipLevel, @@ -1495,9 +1516,9 @@ namespace Diligent BufferVk.SetAccessFlags(NewAccessFlags); } - VulkanDynamicAllocation DeviceContextVkImpl::AllocateDynamicSpace(Uint32 SizeInBytes) + VulkanDynamicAllocation DeviceContextVkImpl::AllocateDynamicSpace(Uint32 SizeInBytes, Uint32 Alignment) { - auto DynAlloc = m_DynamicHeap.Allocate(SizeInBytes, 0); + auto DynAlloc = m_DynamicHeap.Allocate(SizeInBytes, Alignment); #ifdef DEVELOPMENT DynAlloc.dvpFrameNumber = m_ContextFrameNumber; #endif diff --git a/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp index 476766aa..b6ab45bf 100644 --- a/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp +++ b/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp @@ -323,7 +323,7 @@ void RenderDeviceVkImpl::FinishFrame(bool ReleaseAllResources) // until the GPU is finished with the current frame ProcessStaleResources(SubmittedCmdBuffNumber, SubmittedFenceValue, CompletedFenceValue); - m_DynamicMemoryManager.ReleaseStaleAllocations(CompletedFenceValue); + m_DynamicMemoryManager.ReleaseStaleBlocks(CompletedFenceValue); Atomics::AtomicIncrement(m_FrameNumber); } diff --git a/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp index b2487bda..b95094f4 100644 --- a/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp +++ b/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp @@ -150,8 +150,10 @@ TextureVkImpl :: TextureVkImpl(IReferenceCounters* pRefCounters, else ImageMemoryFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + VERIFY( IsPowerOfTwo(MemReqs.alignment), "Alignment is not power of 2!"); m_MemoryAllocation = pRenderDeviceVk->AllocateMemory(MemReqs, ImageMemoryFlags); - auto AlignedOffset = (m_MemoryAllocation.UnalignedOffset + (MemReqs.alignment-1)) & ~(MemReqs.alignment-1); + auto AlignedOffset = Align(m_MemoryAllocation.UnalignedOffset, MemReqs.alignment); + VERIFY_EXPR(m_MemoryAllocation.Size >= MemReqs.size + (AlignedOffset - m_MemoryAllocation.UnalignedOffset)); auto Memory = m_MemoryAllocation.Page->GetVkMemory(); auto err = LogicalDevice.BindImageMemory(m_VulkanImage, Memory, AlignedOffset); CHECK_VK_ERROR_AND_THROW(err, "Failed to bind image memory"); @@ -268,12 +270,14 @@ TextureVkImpl :: TextureVkImpl(IReferenceCounters* pRefCounters, VulkanUtilities::BufferWrapper StagingBuffer = LogicalDevice.CreateBuffer(VkStaginBuffCI, StagingBufferName.c_str()); VkMemoryRequirements StagingBufferMemReqs = LogicalDevice.GetBufferMemoryRequirements(StagingBuffer); + VERIFY( IsPowerOfTwo(StagingBufferMemReqs.alignment), "Alignment is not power of 2!"); // VK_MEMORY_PROPERTY_HOST_COHERENT_BIT bit specifies that the host cache management commands vkFlushMappedMemoryRanges // and vkInvalidateMappedMemoryRanges are NOT needed to flush host writes to the device or make device writes visible // to the host (10.2) auto StagingMemoryAllocation = pRenderDeviceVk->AllocateMemory(StagingBufferMemReqs, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); auto StagingBufferMemory = StagingMemoryAllocation.Page->GetVkMemory(); - auto AlignedStagingMemOffset = (StagingMemoryAllocation.UnalignedOffset + (StagingBufferMemReqs.alignment-1)) & ~(StagingBufferMemReqs.alignment-1); + auto AlignedStagingMemOffset = Align(StagingMemoryAllocation.UnalignedOffset, StagingBufferMemReqs.alignment); + VERIFY_EXPR(StagingMemoryAllocation.Size >= StagingBufferMemReqs.size + (AlignedStagingMemOffset - StagingMemoryAllocation.UnalignedOffset)); auto *StagingData = reinterpret_cast<uint8_t*>(StagingMemoryAllocation.Page->GetCPUMemory()); VERIFY_EXPR(StagingData != nullptr); diff --git a/Graphics/GraphicsEngineVulkan/src/VulkanDynamicHeap.cpp b/Graphics/GraphicsEngineVulkan/src/VulkanDynamicHeap.cpp index 429c174c..e535dcad 100644 --- a/Graphics/GraphicsEngineVulkan/src/VulkanDynamicHeap.cpp +++ b/Graphics/GraphicsEngineVulkan/src/VulkanDynamicHeap.cpp @@ -40,11 +40,11 @@ static VkDeviceSize GetDefaultAlignment(const VulkanUtilities::VulkanPhysicalDev VulkanDynamicMemoryManager::VulkanDynamicMemoryManager(IMemoryAllocator& Allocator, RenderDeviceVkImpl& DeviceVk, Uint32 Size) : + TBase(Allocator, Size), m_DeviceVk(DeviceVk), - m_DefaultAlignment(GetDefaultAlignment(DeviceVk.GetPhysicalDevice())), - m_AllocationStrategy(Allocator, Size) + m_DefaultAlignment(GetDefaultAlignment(DeviceVk.GetPhysicalDevice())) { - VERIFY( (Size & (MinAlignment-1)) == 0, "Heap size is not min aligned"); + VERIFY( (Size & (MasterBlockAlignment-1)) == 0, "Heap size (", Size, " is not aligned by the master block alignment (", MasterBlockAlignment, ")"); VkBufferCreateInfo VkBuffCI = {}; VkBuffCI.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; VkBuffCI.pNext = nullptr; @@ -114,24 +114,27 @@ void VulkanDynamicMemoryManager::Destroy() VulkanDynamicMemoryManager::~VulkanDynamicMemoryManager() { VERIFY(m_BufferMemory == VK_NULL_HANDLE && m_VkBuffer == VK_NULL_HANDLE, "Vulkan resources must be explcitly released with Destroy()"); - auto Size = m_AllocationStrategy.GetSize(); + auto Size = GetSize(); LOG_INFO_MESSAGE("Dynamic memory manager usage stats:\n" " Total size: ", FormatMemorySize(Size, 2), ". Peak allocated size: ", FormatMemorySize(m_TotalPeakSize, 2, Size), ". Peak utilization: ", std::fixed, std::setprecision(1), static_cast<double>(m_TotalPeakSize) / static_cast<double>(std::max(Size, size_t{1})) * 100.0, '%' ); } -VulkanDynamicMemoryManager::OffsetType VulkanDynamicMemoryManager::Allocate(OffsetType SizeInBytes) + +VulkanDynamicMemoryManager::MasterBlock VulkanDynamicMemoryManager::AllocateMasterBlock(OffsetType SizeInBytes, OffsetType Alignment) { - VERIFY( (SizeInBytes & (MinAlignment-1)) == 0, "Allocation size is not minimally aligned" ); - - if (SizeInBytes > m_AllocationStrategy.GetSize()) + if (Alignment == 0) + Alignment = MasterBlockAlignment; + + if (SizeInBytes > GetSize()) { - LOG_ERROR("Requested dynamic allocation size ", SizeInBytes, " exceeds maximum dynamic memory size ", m_AllocationStrategy.GetSize(), ". The app should increase dynamic heap size."); - return AllocationStategy::InvalidOffset; + LOG_ERROR("Requested dynamic allocation size ", SizeInBytes, " exceeds maximum dynamic memory size ", GetSize(), ". The app should increase dynamic heap size."); + return MasterBlock{}; } - auto Offset = m_AllocationStrategy.Allocate(SizeInBytes); - if(Offset == AllocationStategy::InvalidOffset) + + auto Block = TBase::AllocateMasterBlock(SizeInBytes, Alignment); + if (Block.UnalignedOffset == InvalidOffset) { // Allocation failed. Try to wait for GPU to finish pending frames to release some space auto StartIdleTime = std::chrono::high_resolution_clock::now(); @@ -139,12 +142,12 @@ VulkanDynamicMemoryManager::OffsetType VulkanDynamicMemoryManager::Allocate(Offs static constexpr const auto MaxIdleDuration = std::chrono::duration<double>{60.0 / 1000.0}; // 60 ms std::chrono::duration<double> IdleDuration; Uint32 SleepIterations = 0; - while(Offset == AllocationStategy::InvalidOffset && IdleDuration < MaxIdleDuration) + while (Block.UnalignedOffset == InvalidOffset && IdleDuration < MaxIdleDuration) { auto LastCompletedFenceValue = m_DeviceVk.GetCompletedFenceValue(); - m_AllocationStrategy.ReleaseStaleAllocations(LastCompletedFenceValue); - Offset = m_AllocationStrategy.Allocate(SizeInBytes); - if (Offset == AllocationStategy::InvalidOffset) + ReleaseStaleBlocks(LastCompletedFenceValue); + Block = AllocateMasterBlock(SizeInBytes, Alignment); + if (Block.UnalignedOffset == InvalidOffset) { std::this_thread::sleep_for(SleepPeriod); ++SleepIterations; @@ -154,14 +157,14 @@ VulkanDynamicMemoryManager::OffsetType VulkanDynamicMemoryManager::Allocate(Offs IdleDuration = std::chrono::duration_cast<std::chrono::duration<double>>(CurrTime - StartIdleTime); } - if(Offset == AllocationStategy::InvalidOffset) + if (Block.UnalignedOffset == InvalidOffset) { // Last resort - idle GPU (there seems to be a driver bug: vkQueueWaitIdle() deadlocks and never returns) //m_DeviceVk.IdleGPU(true); //auto LastCompletedFenceValue = m_DeviceVk.GetCompletedFenceValue(); //m_AllocationStrategy.ReleaseStaleAllocations(LastCompletedFenceValue); //Offset = m_AllocationStrategy.Allocate(SizeInBytes); - if (Offset == AllocationStategy::InvalidOffset) + if (Block.UnalignedOffset == InvalidOffset) { LOG_ERROR_MESSAGE("Space in dynamic heap is exausted! After idling for ", std::fixed, std::setprecision(1), IdleDuration.count()*1000.0, " ms still no space is available. Increase the size of the heap by setting EngineVkAttribs::DynamicHeapSize to a greater value or optimize dynamic resource usage"); } @@ -183,59 +186,71 @@ VulkanDynamicMemoryManager::OffsetType VulkanDynamicMemoryManager::Allocate(Offs } } - if (Offset != AllocationStategy::InvalidOffset) + if (Block.UnalignedOffset != InvalidOffset) { - m_TotalPeakSize = std::max(m_TotalPeakSize, m_AllocationStrategy.GetUsedSize()); + m_TotalPeakSize = std::max(m_TotalPeakSize, GetUsedSize()); } - return Offset; + + return Block; } VulkanDynamicAllocation VulkanDynamicHeap::Allocate(Uint32 SizeInBytes, Uint32 Alignment) { - if (Alignment == 0) - Alignment = static_cast<Uint32>(m_DynamicMemMgr.m_DefaultAlignment); - - const Uint32 AlignmentMask = Alignment - 1; - // Assert that it's a power of two. - VERIFY_EXPR((AlignmentMask & Alignment) == 0); + VERIFY_EXPR(Alignment > 0); + VERIFY(IsPowerOfTwo(Alignment), "Alignment (", Alignment, ") must be power of 2"); - // Align the allocation - Uint32 AlignedSize = (SizeInBytes + AlignmentMask) & ~AlignmentMask; - - auto Offset = InvalidOffset; - if(AlignedSize > m_PagSize/2) + auto AlignedOffset = InvalidOffset; + OffsetType AlignedSize = 0; + if(SizeInBytes > m_MasterBlockSize/2) { - AlignedSize = (SizeInBytes + VulkanDynamicMemoryManager::MinAlignment) & ~(VulkanDynamicMemoryManager::MinAlignment-1); // Allocate directly from the memory manager - Offset = m_DynamicMemMgr.Allocate(AlignedSize); - m_Allocations.emplace_back(Offset, AlignedSize); + auto MasterBlock = m_DynamicMemMgr.AllocateMasterBlock(SizeInBytes, Alignment); + if (MasterBlock.UnalignedOffset != InvalidOffset) + { + AlignedOffset = Align(MasterBlock.UnalignedOffset, size_t{Alignment}); + AlignedSize = MasterBlock.Size; + VERIFY_EXPR(MasterBlock.Size >= SizeInBytes + (AlignedOffset - MasterBlock.UnalignedOffset)); + m_MasterBlocks.emplace_back(MasterBlock); + } } else { - if(m_CurrOffset == InvalidOffset || AlignedSize > m_AvailableSize) + if (m_CurrOffset == InvalidOffset || SizeInBytes + (Align(m_CurrOffset, size_t{Alignment}) - m_CurrOffset) > m_AvailableSize) { - m_CurrOffset = m_DynamicMemMgr.Allocate(m_PagSize); - m_Allocations.emplace_back(m_CurrOffset, m_PagSize); - m_AvailableSize = m_PagSize; + auto MasterBlock = m_DynamicMemMgr.AllocateMasterBlock(m_MasterBlockSize, 0); + if (MasterBlock.UnalignedOffset != InvalidOffset) + { + m_CurrOffset = MasterBlock.UnalignedOffset; + m_MasterBlocks.emplace_back(MasterBlock); + m_AvailableSize = m_MasterBlockSize; + } } - if(m_CurrOffset != InvalidOffset) + + if (m_CurrOffset != InvalidOffset) { - Offset = m_CurrOffset; - m_AvailableSize -= AlignedSize; - m_CurrOffset += AlignedSize; + AlignedOffset = Align(m_CurrOffset, size_t{Alignment}); + AlignedSize = SizeInBytes + (AlignedOffset - m_CurrOffset); + if (AlignedSize <= m_AvailableSize) + { + m_AvailableSize -= static_cast<Uint32>(AlignedSize); + m_CurrOffset += static_cast<Uint32>(AlignedSize); + } + else + AlignedOffset = InvalidOffset; } } // Every device context uses its own dynamic heap, so there is no need to lock - if(Offset != InvalidOffset) + if(AlignedOffset != InvalidOffset) { - m_CurrAllocatedSize += AlignedSize; + m_CurrAllocatedSize += static_cast<Uint32>(AlignedSize); m_CurrUsedSize += SizeInBytes; m_PeakAllocatedSize = std::max(m_PeakAllocatedSize, m_CurrAllocatedSize); m_PeakUsedSize = std::max(m_PeakUsedSize, m_CurrUsedSize); - - return VulkanDynamicAllocation{ m_DynamicMemMgr, Offset, SizeInBytes }; + + VERIFY_EXPR((AlignedOffset & (Alignment-1)) == 0); + return VulkanDynamicAllocation{ m_DynamicMemMgr, AlignedOffset, SizeInBytes }; } else return VulkanDynamicAllocation{}; @@ -243,8 +258,8 @@ VulkanDynamicAllocation VulkanDynamicHeap::Allocate(Uint32 SizeInBytes, Uint32 A void VulkanDynamicHeap::FinishFrame(Uint64 FenceValue) { - m_DynamicMemMgr.DiscardAllocations(m_Allocations, FenceValue); - m_Allocations.clear(); + m_DynamicMemMgr.DiscardMasterBlocks(m_MasterBlocks, FenceValue); + m_MasterBlocks.clear(); m_CurrOffset = InvalidOffset; m_AvailableSize = 0; diff --git a/Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp b/Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp index aba420e8..a41a010f 100644 --- a/Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp +++ b/Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp @@ -81,12 +81,7 @@ VulkanUploadHeap::UploadPageInfo VulkanUploadHeap::CreateNewPage(VkDeviceSize Si VulkanUploadAllocation VulkanUploadHeap::Allocate(size_t SizeInBytes, size_t Alignment) { - VERIFY((Alignment & (Alignment-1)) == 0, "Alignment (", Alignment, ") must be power of two"); - if (Alignment == 0) - { - static constexpr const size_t MinimumAlignment = 4; - Alignment = MinimumAlignment; - } + VERIFY(IsPowerOfTwo(Alignment), "Alignment (", Alignment, ") must be power of two"); VulkanUploadAllocation Allocation; if(SizeInBytes >= m_PageSize/2) @@ -103,7 +98,7 @@ VulkanUploadAllocation VulkanUploadHeap::Allocate(size_t SizeInBytes, size_t Ali } else { - auto AlignmentOffset = ((m_CurrPage.CurrOffset + (Alignment-1)) & ~(Alignment-1)) - m_CurrPage.CurrOffset; + auto AlignmentOffset = Align(m_CurrPage.CurrOffset, Alignment) - m_CurrPage.CurrOffset; if(m_CurrPage.AvailableSize < SizeInBytes + AlignmentOffset) { // Allocate new page diff --git a/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanMemoryManager.cpp b/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanMemoryManager.cpp index b4660819..973b1add 100644 --- a/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanMemoryManager.cpp +++ b/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanMemoryManager.cpp @@ -32,7 +32,7 @@ VulkanMemoryAllocation::~VulkanMemoryAllocation() { if (Page != nullptr) { - Page->Free(*this); + Page->Free(std::move(*this)); } } @@ -75,13 +75,16 @@ VulkanMemoryPage::~VulkanMemoryPage() VERIFY(IsEmpty(), "Destroying a page with not all allocations released"); } -VulkanMemoryAllocation VulkanMemoryPage::Allocate(VkDeviceSize size) +VulkanMemoryAllocation VulkanMemoryPage::Allocate(VkDeviceSize size, VkDeviceSize alignment) { std::lock_guard<std::mutex> Lock(m_Mutex); - auto Offset = m_AllocationMgr.Allocate(size); - if (Offset != Diligent::VariableSizeAllocationsManager::InvalidOffset) + auto Allocation = m_AllocationMgr.Allocate(size, alignment); + if (Allocation.UnalignedOffset != Diligent::VariableSizeAllocationsManager::InvalidOffset) { - return VulkanMemoryAllocation{this, Offset, size}; + // Offset may not necessarily be aligned, but the allocation is guaranteed to be large enough + // to accomodate requested alignment + VERIFY_EXPR( Diligent::Align(Allocation.UnalignedOffset, alignment) - Allocation.UnalignedOffset + size <= Allocation.Size ); + return VulkanMemoryAllocation{this, Allocation.UnalignedOffset, Allocation.Size}; } else { @@ -89,11 +92,12 @@ VulkanMemoryAllocation VulkanMemoryPage::Allocate(VkDeviceSize size) } } -void VulkanMemoryPage::Free(VulkanMemoryAllocation& Allocation) +void VulkanMemoryPage::Free(VulkanMemoryAllocation&& Allocation) { m_ParentMemoryMgr.OnFreeAllocation(Allocation.Size, m_CPUMemory != nullptr); std::lock_guard<std::mutex> Lock(m_Mutex); m_AllocationMgr.Free(Allocation.UnalignedOffset, Allocation.Size); + Allocation = VulkanMemoryAllocation{}; } VulkanMemoryAllocation VulkanMemoryManager::Allocate(const VkMemoryRequirements& MemReqs, VkMemoryPropertyFlags MemoryProps) @@ -130,14 +134,13 @@ VulkanMemoryAllocation VulkanMemoryManager::Allocate(const VkMemoryRequirements& VulkanMemoryAllocation VulkanMemoryManager::Allocate(VkDeviceSize Size, VkDeviceSize Alignment, uint32_t MemoryTypeIndex, bool HostVisible) { - Size += Alignment; VulkanMemoryAllocation Allocation; std::lock_guard<std::mutex> Lock(m_PagesMtx); auto range = m_Pages.equal_range(MemoryTypeIndex); for(auto page_it = range.first; page_it != range.second; ++page_it) { - Allocation = page_it->second.Allocate(Size); + Allocation = page_it->second.Allocate(Size, Alignment); if (Allocation.Page != nullptr) break; } @@ -157,11 +160,16 @@ VulkanMemoryAllocation VulkanMemoryManager::Allocate(VkDeviceSize Size, VkDevice " page. (", Diligent::FormatMemorySize(PageSize, 2), ", type idx: ", MemoryTypeIndex, "). Current allocated size: ", Diligent::FormatMemorySize(m_CurrAllocatedSize[stat_ind], 2)); OnNewPageCreated(it->second); - Allocation = it->second.Allocate(Size); - VERIFY(Allocation.Page != nullptr, "Failed to allocate new memory page"); + Allocation = it->second.Allocate(Size, Alignment); + DEV_CHECK_ERR(Allocation.Page != nullptr, "Failed to allocate new memory page"); } - m_CurrUsedSize[stat_ind].fetch_add(Size); + if (Allocation.Page != nullptr) + { + VERIFY_EXPR(Size + Diligent::Align(Allocation.UnalignedOffset, Alignment) - Allocation.UnalignedOffset <= Allocation.Size); + } + + m_CurrUsedSize[stat_ind].fetch_add(Allocation.Size); m_PeakUsedSize[stat_ind] = std::max(m_PeakUsedSize[stat_ind], static_cast<VkDeviceSize>(m_CurrUsedSize[stat_ind].load())); return Allocation; |
