summaryrefslogtreecommitdiffstats
path: root/Graphics/GraphicsEngineVulkan
diff options
context:
space:
mode:
authorEgor Yusov <egor.yusov@gmail.com>2018-09-09 03:13:04 +0000
committerEgor Yusov <egor.yusov@gmail.com>2018-09-09 03:13:04 +0000
commitd7dda5023e6dfa999286a45ed314158330c18a5f (patch)
treef01498bdad1032243bd7a97732eb9284152c6d60 /Graphics/GraphicsEngineVulkan
parentFixed debug check in ShaderResourceBindingD3D12Impl::InitializeStaticResources() (diff)
downloadDiligentCore-d7dda5023e6dfa999286a45ed314158330c18a5f.tar.gz
DiligentCore-d7dda5023e6dfa999286a45ed314158330c18a5f.zip
Added alignment parameter to allocation managers (fixed https://github.com/DiligentGraphics/DiligentCore/issues/29)
Diffstat (limited to 'Graphics/GraphicsEngineVulkan')
-rw-r--r--Graphics/GraphicsEngineVulkan/include/BufferVkImpl.h5
-rw-r--r--Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h14
-rw-r--r--Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h6
-rw-r--r--Graphics/GraphicsEngineVulkan/include/VulkanDynamicHeap.h178
-rw-r--r--Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanMemoryManager.h4
-rw-r--r--Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanPhysicalDevice.h10
-rw-r--r--Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp39
-rw-r--r--Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp41
-rw-r--r--Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp2
-rw-r--r--Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp8
-rw-r--r--Graphics/GraphicsEngineVulkan/src/VulkanDynamicHeap.cpp113
-rw-r--r--Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp9
-rw-r--r--Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanMemoryManager.cpp30
13 files changed, 201 insertions, 258 deletions
diff --git a/Graphics/GraphicsEngineVulkan/include/BufferVkImpl.h b/Graphics/GraphicsEngineVulkan/include/BufferVkImpl.h
index 72b5e217..397a4198 100644
--- a/Graphics/GraphicsEngineVulkan/include/BufferVkImpl.h
+++ b/Graphics/GraphicsEngineVulkan/include/BufferVkImpl.h
@@ -87,7 +87,7 @@ public:
DvpVerifyDynamicAllocation(pCtx);
#endif
auto& DynAlloc = m_DynamicAllocations[CtxId];
- return static_cast<Uint32>(DynAlloc.Offset);
+ return static_cast<Uint32>(DynAlloc.AlignedOffset);
}
}
VkBuffer GetVkBuffer()const override final;
@@ -117,7 +117,8 @@ private:
virtual void CreateViewInternal( const struct BufferViewDesc& ViewDesc, IBufferView** ppView, bool bIsDefaultView )override;
VulkanUtilities::BufferViewWrapper CreateView(struct BufferViewDesc &ViewDesc);
- VkAccessFlags m_AccessFlags = 0;
+ VkAccessFlags m_AccessFlags = 0;
+ Uint32 m_DynamicOffsetAlignment = 0;
#ifdef DEVELOPMENT
std::vector< std::pair<MAP_TYPE, Uint32> > m_DvpMapType;
diff --git a/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h b/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h
index 050e0103..06a6a744 100644
--- a/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h
+++ b/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h
@@ -172,7 +172,7 @@ public:
return m_DynamicDescriptorPool.Allocate(SetLayout);
}
- VulkanDynamicAllocation AllocateDynamicSpace(Uint32 SizeInBytes);
+ VulkanDynamicAllocation AllocateDynamicSpace(Uint32 SizeInBytes, Uint32 Alignment);
void ResetRenderTargets();
Int64 GetContextFrameNumber()const{return m_ContextFrameNumber;}
@@ -191,11 +191,12 @@ private:
struct BufferToTextureCopyInfo
{
- Uint32 RowSize = 0;
- Uint32 Stride = 0;
- Uint32 DepthStride = 0;
- Uint32 MemorySize = 0;
- Uint32 RowCount = 0;
+ Uint32 RowSize = 0;
+ Uint32 Stride = 0;
+ Uint32 StrideInTexels = 0;
+ Uint32 DepthStride = 0;
+ Uint32 MemorySize = 0;
+ Uint32 RowCount = 0;
Box Region;
};
BufferToTextureCopyInfo GetBufferToTextureCopyInfo(const TextureDesc& TexDesc,
@@ -204,6 +205,7 @@ private:
void CopyBufferToTexture(VkBuffer vkBuffer,
Uint32 BufferOffset,
+ Uint32 BufferRowStrideInTexels,
const Box& Region,
TextureVkImpl& TextureVk,
Uint32 MipLevel,
diff --git a/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h b/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h
index 227526b2..49db82b9 100644
--- a/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h
+++ b/Graphics/GraphicsEngineVulkan/include/RenderDeviceVkImpl.h
@@ -125,10 +125,10 @@ public:
}
std::shared_ptr<const VulkanUtilities::VulkanInstance> GetVulkanInstance()const{return m_VulkanInstance;}
- const VulkanUtilities::VulkanPhysicalDevice& GetPhysicalDevice(){return *m_PhysicalDevice;}
- const VulkanUtilities::VulkanLogicalDevice& GetLogicalDevice() {return *m_LogicalVkDevice;}
+ const VulkanUtilities::VulkanPhysicalDevice& GetPhysicalDevice()const {return *m_PhysicalDevice;}
+ const VulkanUtilities::VulkanLogicalDevice& GetLogicalDevice () {return *m_LogicalVkDevice;}
FramebufferCache& GetFramebufferCache(){return m_FramebufferCache;}
- RenderPassCache& GetRenderPassCache(){return m_RenderPassCache;}
+ RenderPassCache& GetRenderPassCache() {return m_RenderPassCache;}
VulkanUtilities::VulkanMemoryAllocation AllocateMemory(const VkMemoryRequirements& MemReqs, VkMemoryPropertyFlags MemoryProperties)
{
diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanDynamicHeap.h b/Graphics/GraphicsEngineVulkan/include/VulkanDynamicHeap.h
index 3e3d0c7f..79a95abe 100644
--- a/Graphics/GraphicsEngineVulkan/include/VulkanDynamicHeap.h
+++ b/Graphics/GraphicsEngineVulkan/include/VulkanDynamicHeap.h
@@ -25,10 +25,10 @@
#include <mutex>
#include "Vulkan.h"
-#include "RingBuffer.h"
#include "VulkanUtilities/VulkanMemoryManager.h"
#include "VulkanUtilities/VulkanLogicalDevice.h"
#include "VulkanUtilities/VulkanObjectWrappers.h"
+#include "DynamicHeap.h"
namespace Diligent
{
@@ -47,9 +47,9 @@ struct VulkanDynamicAllocation
{
VulkanDynamicAllocation()noexcept{}
- VulkanDynamicAllocation(VulkanDynamicMemoryManager& _DynamicMemMgr, size_t _Offset, size_t _Size)noexcept :
+ VulkanDynamicAllocation(VulkanDynamicMemoryManager& _DynamicMemMgr, size_t _AlignedOffset, size_t _Size)noexcept :
pDynamicMemMgr(&_DynamicMemMgr),
- Offset (_Offset),
+ AlignedOffset (_AlignedOffset),
Size (_Size)
{}
@@ -57,15 +57,15 @@ struct VulkanDynamicAllocation
VulkanDynamicAllocation& operator = (const VulkanDynamicAllocation&) = delete;
VulkanDynamicAllocation (VulkanDynamicAllocation&& rhs)noexcept :
pDynamicMemMgr(rhs.pDynamicMemMgr),
- Offset (rhs.Offset),
+ AlignedOffset (rhs.AlignedOffset),
Size (rhs.Size)
#ifdef DEVELOPMENT
, dvpFrameNumber(rhs.dvpFrameNumber)
#endif
{
rhs.pDynamicMemMgr = nullptr;
- rhs.Offset = 0;
- rhs.Size = 0;
+ rhs.AlignedOffset = 0;
+ rhs.Size = 0;
#ifdef DEVELOPMENT
rhs.dvpFrameNumber = 0;
#endif
@@ -74,20 +74,20 @@ struct VulkanDynamicAllocation
VulkanDynamicAllocation& operator = (VulkanDynamicAllocation&& rhs)noexcept // Must be noexcept on MSVC, so can't use = default
{
pDynamicMemMgr = rhs.pDynamicMemMgr;
- Offset = rhs.Offset;
+ AlignedOffset = rhs.AlignedOffset;
Size = rhs.Size;
rhs.pDynamicMemMgr = nullptr;
- rhs.Offset = 0;
+ rhs.AlignedOffset = 0;
rhs.Size = 0;
#ifdef DEVELOPMENT
- dvpFrameNumber = rhs.dvpFrameNumber;
+ dvpFrameNumber = rhs.dvpFrameNumber;
rhs.dvpFrameNumber = 0;
#endif
return *this;
}
VulkanDynamicMemoryManager* pDynamicMemMgr = nullptr;
- size_t Offset = 0; // Offset from the start of the buffer resource
+ size_t AlignedOffset = 0; // Offset from the start of the buffer
size_t Size = 0; // Reserved size of this allocation
#ifdef DEVELOPMENT
Int64 dvpFrameNumber = 0;
@@ -95,159 +95,36 @@ struct VulkanDynamicAllocation
};
-// Having global ring buffer shared between all contexts is inconvinient because all contexts
-// must share the same frame. Having individual ring bufer per context may result in a lot of unused
-// memory. As a result, ring buffer is not currently used for dynamic memory management.
-// Instead, every dynamic heap allocates pages from the global dynamic memory manager.
-class RingBufferAllocationStrategy
-{
-public:
- using OffsetType = RingBuffer::OffsetType;
- static constexpr const OffsetType InvalidOffset = RingBuffer::InvalidOffset;
-
- RingBufferAllocationStrategy(IMemoryAllocator& Allocator,
- Uint32 Size) :
- m_RingBuffer(Size, Allocator)
- {}
-
- RingBufferAllocationStrategy (const RingBufferAllocationStrategy&) = delete;
- RingBufferAllocationStrategy (RingBufferAllocationStrategy&&) = delete;
- RingBufferAllocationStrategy& operator= (const RingBufferAllocationStrategy&) = delete;
- RingBufferAllocationStrategy& operator= (RingBufferAllocationStrategy&&) = delete;
-
- void DiscardAllocations(const std::vector<std::pair<OffsetType, OffsetType>>& Allocations, Uint64 FenceValue)
- {
- std::lock_guard<std::mutex> Lock(m_RingBufferMtx);
- m_RingBuffer.FinishCurrentFrame(FenceValue);
- }
-
- void ReleaseStaleAllocations(Uint64 LastCompletedFenceValue)
- {
- std::lock_guard<std::mutex> Lock(m_RingBufferMtx);
- m_RingBuffer.ReleaseCompletedFrames(LastCompletedFenceValue);
- }
-
- OffsetType Allocate(OffsetType SizeInBytes)
- {
- std::lock_guard<std::mutex> Lock(m_RingBufferMtx);
- return m_RingBuffer.Allocate(SizeInBytes);
- }
-
- OffsetType GetSize() const{return m_RingBuffer.GetMaxSize();}
- OffsetType GetUsedSize()const{return m_RingBuffer.GetUsedSize();}
-private:
- std::mutex m_RingBufferMtx;
- RingBuffer m_RingBuffer;
-};
-
-
-class ListBasedAllocationStrategy
-{
-public:
- using OffsetType = VariableSizeAllocationsManager::OffsetType;
- static constexpr const OffsetType InvalidOffset = VariableSizeAllocationsManager::InvalidOffset;
-
- ListBasedAllocationStrategy(IMemoryAllocator& Allocator,
- Uint32 Size) :
- m_AllocationsMgr(Size, Allocator)
- {}
-
- ListBasedAllocationStrategy (const ListBasedAllocationStrategy&) = delete;
- ListBasedAllocationStrategy (ListBasedAllocationStrategy&&) = delete;
- ListBasedAllocationStrategy& operator= (const ListBasedAllocationStrategy&) = delete;
- ListBasedAllocationStrategy& operator= (ListBasedAllocationStrategy&&) = delete;
-
- void DiscardAllocations(const std::vector<std::pair<OffsetType, OffsetType>>& Allocations, Uint64 FenceValue)
- {
- std::lock_guard<std::mutex> Lock(m_ReleaseQueueMtx);
- for(const auto& Allocation : Allocations)
- m_ReleaseQueue.emplace_back(Allocation.first, Allocation.second, FenceValue);
- }
-
- void ReleaseStaleAllocations(Uint64 LastCompletedFenceValue)
- {
- std::lock_guard<std::mutex> MgrLock(m_AllocationsMgrMtx);
- std::lock_guard<std::mutex> QueueLock(m_ReleaseQueueMtx);
- while (!m_ReleaseQueue.empty())
- {
- auto &FirstAllocation = m_ReleaseQueue.front();
- if (FirstAllocation.FenceValue <= LastCompletedFenceValue)
- {
- m_AllocationsMgr.Free(FirstAllocation.Offset, FirstAllocation.Size);
- m_ReleaseQueue.pop_front();
- }
- else
- break;
- }
- }
-
- OffsetType Allocate(OffsetType SizeInBytes)
- {
- std::lock_guard<std::mutex> Lock(m_AllocationsMgrMtx);
- return m_AllocationsMgr.Allocate(SizeInBytes);
- }
- OffsetType GetSize() const{return m_AllocationsMgr.GetMaxSize();}
- OffsetType GetUsedSize()const{return m_AllocationsMgr.GetUsedSize();}
-
-private:
- std::mutex m_AllocationsMgrMtx;
- VariableSizeAllocationsManager m_AllocationsMgr;
-
- struct StaleAllocationInfo
- {
- const OffsetType Offset;
- const OffsetType Size;
- const Uint64 FenceValue;
- StaleAllocationInfo(OffsetType _Offset,
- OffsetType _Size,
- Uint64 _FenceValue) :
- Offset (_Offset),
- Size (_Size),
- FenceValue(_FenceValue)
- {}
- };
- std::deque< StaleAllocationInfo > m_ReleaseQueue;
- std::mutex m_ReleaseQueueMtx;
-};
// We cannot use global memory manager for dynamic resources because they
// need to use the same Vulkan buffer
-class VulkanDynamicMemoryManager
+class VulkanDynamicMemoryManager : public DynamicHeap::MasterBlockListBasedManager // or MasterBlockRingBufferBasedManager
{
public:
- using AllocationStategy = ListBasedAllocationStrategy; // or RingBufferAllocationStrategy
- using OffsetType = AllocationStategy::OffsetType;
+ using TBase = DynamicHeap::MasterBlockListBasedManager;
+ using OffsetType = TBase::OffsetType;
+ using TBase::InvalidOffset;
+ using MasterBlock = TBase::MasterBlock;
VulkanDynamicMemoryManager(IMemoryAllocator& Allocator,
class RenderDeviceVkImpl& DeviceVk,
Uint32 Size);
~VulkanDynamicMemoryManager();
- VulkanDynamicMemoryManager (const VulkanDynamicMemoryManager&) = delete;
- VulkanDynamicMemoryManager (VulkanDynamicMemoryManager&&) = delete;
- VulkanDynamicMemoryManager& operator= (const VulkanDynamicMemoryManager&) = delete;
- VulkanDynamicMemoryManager& operator= (VulkanDynamicMemoryManager&&) = delete;
-
- void ReleaseStaleAllocations(Uint64 LastCompletedFenceValue)
- {
- m_AllocationStrategy.ReleaseStaleAllocations(LastCompletedFenceValue);
- }
-
- void DiscardAllocations(const std::vector<std::pair<OffsetType, OffsetType>>& Allocations, Uint64 FenceValue)
- {
- m_AllocationStrategy.DiscardAllocations(Allocations, FenceValue);
- }
+ VulkanDynamicMemoryManager (const VulkanDynamicMemoryManager&) = delete;
+ VulkanDynamicMemoryManager ( VulkanDynamicMemoryManager&&) = delete;
+ VulkanDynamicMemoryManager& operator= (const VulkanDynamicMemoryManager&) = delete;
+ VulkanDynamicMemoryManager& operator= ( VulkanDynamicMemoryManager&&) = delete;
VkBuffer GetVkBuffer() const{return m_VkBuffer;}
Uint8* GetCPUAddress()const{return m_CPUAddress;}
void Destroy();
- static constexpr const Uint32 MinAlignment = 1024;
+ static constexpr const Uint32 MasterBlockAlignment = 1024;
+ MasterBlock AllocateMasterBlock(OffsetType SizeInBytes, OffsetType Alignment);
private:
- friend class VulkanDynamicHeap;
- OffsetType Allocate(OffsetType SizeInBytes);
RenderDeviceVkImpl& m_DeviceVk;
VulkanUtilities::BufferWrapper m_VkBuffer;
@@ -255,8 +132,6 @@ private:
Uint8* m_CPUAddress;
const VkDeviceSize m_DefaultAlignment;
- AllocationStategy m_AllocationStrategy;
-
OffsetType m_TotalPeakSize = 0;
};
@@ -267,7 +142,7 @@ public:
VulkanDynamicHeap(VulkanDynamicMemoryManager& DynamicMemMgr, std::string HeapName, Uint32 PageSize) :
m_DynamicMemMgr(DynamicMemMgr),
m_HeapName(std::move(HeapName)),
- m_PagSize(PageSize)
+ m_MasterBlockSize(PageSize)
{}
VulkanDynamicHeap (const VulkanDynamicHeap&) = delete;
@@ -280,17 +155,18 @@ public:
VulkanDynamicAllocation Allocate(Uint32 SizeInBytes, Uint32 Alignment);
void FinishFrame(Uint64 FenceValue);
- using OffsetType = VulkanDynamicMemoryManager::AllocationStategy::OffsetType;
- static constexpr OffsetType InvalidOffset = VulkanDynamicMemoryManager::AllocationStategy::InvalidOffset;
+ using OffsetType = VulkanDynamicMemoryManager::OffsetType;
+ using MasterBlock = VulkanDynamicMemoryManager::MasterBlock;
+ static constexpr OffsetType InvalidOffset = VulkanDynamicMemoryManager::InvalidOffset;
private:
VulkanDynamicMemoryManager& m_DynamicMemMgr;
const std::string m_HeapName;
- std::vector<std::pair<OffsetType, OffsetType>> m_Allocations;
+ std::vector<MasterBlock> m_MasterBlocks;
OffsetType m_CurrOffset = InvalidOffset;
- const Uint32 m_PagSize;
+ const Uint32 m_MasterBlockSize;
Uint32 m_AvailableSize = 0;
Uint32 m_CurrAllocatedSize = 0;
diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanMemoryManager.h b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanMemoryManager.h
index 270a3030..bb0f4f59 100644
--- a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanMemoryManager.h
+++ b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanMemoryManager.h
@@ -112,7 +112,7 @@ public:
VkDeviceSize GetPageSize()const{return m_AllocationMgr.GetMaxSize();}
VkDeviceSize GetUsedSize()const{return m_AllocationMgr.GetUsedSize();}
- VulkanMemoryAllocation Allocate(VkDeviceSize size);
+ VulkanMemoryAllocation Allocate(VkDeviceSize size, VkDeviceSize alignment);
VkDeviceMemory GetVkMemory()const{return m_VkMemory;}
void* GetCPUMemory()const{return m_CPUMemory;}
@@ -121,7 +121,7 @@ private:
friend struct VulkanMemoryAllocation;
// Memory is reclaimed immediately. The application is responsible to ensure it is not in use by the GPU
- void Free(VulkanMemoryAllocation& Allocation);
+ void Free(VulkanMemoryAllocation&& Allocation);
VulkanMemoryManager& m_ParentMemoryMgr;
std::mutex m_Mutex;
diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanPhysicalDevice.h b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanPhysicalDevice.h
index 5e67138e..86dee23c 100644
--- a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanPhysicalDevice.h
+++ b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanPhysicalDevice.h
@@ -39,14 +39,14 @@ namespace VulkanUtilities
static std::unique_ptr<VulkanPhysicalDevice> Create(VkPhysicalDevice vkDevice);
- uint32_t FindQueueFamily(VkQueueFlags QueueFlags)const;
- VkPhysicalDevice GetVkDeviceHandle()const{return m_VkDevice;}
- bool IsExtensionSupported(const char* ExtensionName)const;
- bool CheckPresentSupport(uint32_t queueFamilyIndex, VkSurfaceKHR VkSurface)const;
+ uint32_t FindQueueFamily (VkQueueFlags QueueFlags) const;
+ VkPhysicalDevice GetVkDeviceHandle () const { return m_VkDevice; }
+ bool IsExtensionSupported(const char* ExtensionName) const;
+ bool CheckPresentSupport (uint32_t queueFamilyIndex, VkSurfaceKHR VkSurface) const;
static constexpr uint32_t InvalidMemoryTypeIndex = static_cast<uint32_t>(-1);
uint32_t GetMemoryTypeIndex(uint32_t typeBits, VkMemoryPropertyFlags properties)const;
- const VkPhysicalDeviceProperties& GetProperties()const{return m_Properties;}
+ const VkPhysicalDeviceProperties& GetProperties() const {return m_Properties;}
private:
VulkanPhysicalDevice(VkPhysicalDevice vkDevice);
diff --git a/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp
index ce0e8f4b..ba9c2ca6 100644
--- a/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/BufferVkImpl.cpp
@@ -69,6 +69,8 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters* pRefCounters,
}
const auto& LogicalDevice = pRenderDeviceVk->GetLogicalDevice();
+ const auto& DeviceLimits = pRenderDeviceVk->GetPhysicalDevice().GetProperties().limits;
+ m_DynamicOffsetAlignment = std::max(Uint32{4}, static_cast<Uint32>(DeviceLimits.optimalBufferCopyOffsetAlignment));
VkBufferCreateInfo VkBuffCI = {};
VkBuffCI.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO;
@@ -95,6 +97,12 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters* pRefCounters,
//
// So we have to set both VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT and VK_BUFFER_USAGE_STORAGE_BUFFER_BIT bits
VkBuffCI.usage |= VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
+
+ // Each element of pDynamicOffsets of vkCmdBindDescriptorSets function which corresponds to a descriptor
+ // binding with type VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC must be a multiple of
+ // VkPhysicalDeviceLimits::minStorageBufferOffsetAlignment (13.2.5)
+ m_DynamicOffsetAlignment = std::max(m_DynamicOffsetAlignment, static_cast<Uint32>(DeviceLimits.minTexelBufferOffsetAlignment));
+ m_DynamicOffsetAlignment = std::max(m_DynamicOffsetAlignment, static_cast<Uint32>(DeviceLimits.minStorageBufferOffsetAlignment));
}
if (m_Desc.BindFlags & BIND_SHADER_RESOURCE)
{
@@ -102,6 +110,9 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters* pRefCounters,
// HLSL buffer SRV are mapped to storge buffers in GLSL, so we need to set both
// VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER and VK_BUFFER_USAGE_STORAGE_BUFFER_BIT flags
VkBuffCI.usage |= VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
+
+ m_DynamicOffsetAlignment = std::max(m_DynamicOffsetAlignment, static_cast<Uint32>(DeviceLimits.minTexelBufferOffsetAlignment));
+ m_DynamicOffsetAlignment = std::max(m_DynamicOffsetAlignment, static_cast<Uint32>(DeviceLimits.minStorageBufferOffsetAlignment));
}
if (m_Desc.BindFlags & BIND_VERTEX_BUFFER)
VkBuffCI.usage |= VK_BUFFER_USAGE_VERTEX_BUFFER_BIT;
@@ -110,8 +121,15 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters* pRefCounters,
if (m_Desc.BindFlags & BIND_INDIRECT_DRAW_ARGS)
VkBuffCI.usage |= VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT;
if (m_Desc.BindFlags & BIND_UNIFORM_BUFFER)
+ {
VkBuffCI.usage |= VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT;
+ // Each element of pDynamicOffsets parameter of vkCmdBindDescriptorSets function which corresponds to a descriptor
+ // binding with type VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC must be a multiple of
+ // VkPhysicalDeviceLimits::minUniformBufferOffsetAlignment (13.2.5)
+ m_DynamicOffsetAlignment = std::max(m_DynamicOffsetAlignment, static_cast<Uint32>(DeviceLimits.minUniformBufferOffsetAlignment));
+ }
+
if(m_Desc.Usage == USAGE_DYNAMIC)
{
auto CtxCount = 1 + pRenderDeviceVk->GetNumDeferredContexts();
@@ -148,10 +166,11 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters* pRefCounters,
else
BufferMemoryFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
+ VERIFY( IsPowerOfTwo(MemReqs.alignment), "Alignment is not power of 2!");
m_MemoryAllocation = pRenderDeviceVk->AllocateMemory(MemReqs, BufferMemoryFlags);
- VERIFY( (MemReqs.alignment & (MemReqs.alignment-1)) == 0, "Alignment is not power of 2!");
- auto AlignedOffset = (m_MemoryAllocation.UnalignedOffset + (MemReqs.alignment-1)) & ~(MemReqs.alignment-1);
+ auto AlignedOffset = Align(m_MemoryAllocation.UnalignedOffset, MemReqs.alignment);
+ VERIFY(m_MemoryAllocation.Size >= MemReqs.size + (AlignedOffset - m_MemoryAllocation.UnalignedOffset), "Size of memory allocation is too small");
auto Memory = m_MemoryAllocation.Page->GetVkMemory();
auto err = LogicalDevice.BindBufferMemory(m_VulkanBuffer, Memory, AlignedOffset);
CHECK_VK_ERROR_AND_THROW(err, "Failed to bind buffer memory");
@@ -168,15 +187,17 @@ BufferVkImpl :: BufferVkImpl(IReferenceCounters* pRefCounters,
VulkanUtilities::BufferWrapper StagingBuffer = LogicalDevice.CreateBuffer(VkStaginBuffCI, StagingBufferName.c_str());
VkMemoryRequirements StagingBufferMemReqs = LogicalDevice.GetBufferMemoryRequirements(StagingBuffer);
+ VERIFY( IsPowerOfTwo(StagingBufferMemReqs.alignment), "Alignment is not power of 2!");
// VK_MEMORY_PROPERTY_HOST_COHERENT_BIT bit specifies that the host cache management commands vkFlushMappedMemoryRanges
// and vkInvalidateMappedMemoryRanges are NOT needed to flush host writes to the device or make device writes visible
// to the host (10.2)
auto StagingMemoryAllocation = pRenderDeviceVk->AllocateMemory(StagingBufferMemReqs, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
auto StagingBufferMemory = StagingMemoryAllocation.Page->GetVkMemory();
- auto AlignedStagingMemOffset = (StagingMemoryAllocation.UnalignedOffset + (StagingBufferMemReqs.alignment-1)) & ~(StagingBufferMemReqs.alignment-1);
+ auto AlignedStagingMemOffset = Align(StagingMemoryAllocation.UnalignedOffset, StagingBufferMemReqs.alignment);
+ VERIFY_EXPR(StagingMemoryAllocation.Size >= StagingBufferMemReqs.size + (AlignedStagingMemOffset - StagingMemoryAllocation.UnalignedOffset));
- auto *StagingData = reinterpret_cast<uint8_t*>(StagingMemoryAllocation.Page->GetCPUMemory());
+ auto* StagingData = reinterpret_cast<uint8_t*>(StagingMemoryAllocation.Page->GetCPUMemory());
if (StagingData == nullptr)
LOG_BUFFER_ERROR_AND_THROW("Failed to allocate staging data");
memcpy(StagingData + AlignedStagingMemOffset, BuffData.pData, BuffData.DataSize);
@@ -334,7 +355,7 @@ void BufferVkImpl :: Map(IDeviceContext* pContext, MAP_TYPE MapType, Uint32 MapF
auto& DynAllocation = m_DynamicAllocations[pDeviceContextVk->GetContextId()];
if ( (MapFlags & MAP_FLAG_DISCARD) != 0 || DynAllocation.pDynamicMemMgr == nullptr )
{
- DynAllocation = pDeviceContextVk->AllocateDynamicSpace(m_Desc.uiSizeInBytes);
+ DynAllocation = pDeviceContextVk->AllocateDynamicSpace(m_Desc.uiSizeInBytes, m_DynamicOffsetAlignment);
}
else
{
@@ -345,7 +366,7 @@ void BufferVkImpl :: Map(IDeviceContext* pContext, MAP_TYPE MapType, Uint32 MapF
if (DynAllocation.pDynamicMemMgr != nullptr)
{
auto* CPUAddress = DynAllocation.pDynamicMemMgr->GetCPUAddress();
- pMappedData = CPUAddress + DynAllocation.Offset;
+ pMappedData = CPUAddress + DynAllocation.AlignedOffset;
}
else
{
@@ -415,11 +436,11 @@ void BufferVkImpl::Unmap( IDeviceContext* pContext, MAP_TYPE MapType, Uint32 Map
else if (m_Desc.Usage == USAGE_DYNAMIC)
{
VERIFY( MapFlags & (MAP_FLAG_DISCARD | MAP_FLAG_DO_NOT_SYNCHRONIZE), "Vk buffer must be mapped for writing with MAP_FLAG_DISCARD or MAP_FLAG_DO_NOT_SYNCHRONIZE flag");
- if(m_VulkanBuffer != VK_NULL_HANDLE)
+ if (m_VulkanBuffer != VK_NULL_HANDLE)
{
- auto &DynAlloc = m_DynamicAllocations[CtxId];
+ auto& DynAlloc = m_DynamicAllocations[CtxId];
auto vkSrcBuff = DynAlloc.pDynamicMemMgr->GetVkBuffer();
- pDeviceContextVk->UpdateBufferRegion(this, 0, m_Desc.uiSizeInBytes, vkSrcBuff, DynAlloc.Offset);
+ pDeviceContextVk->UpdateBufferRegion(this, 0, m_Desc.uiSizeInBytes, vkSrcBuff, DynAlloc.AlignedOffset);
}
}
}
diff --git a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
index 1a0a3d41..9be700a1 100644
--- a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
@@ -1074,7 +1074,9 @@ namespace Diligent
#endif
VERIFY_EXPR( static_cast<size_t>(NumBytes) == NumBytes );
- auto TmpSpace = m_UploadHeap.Allocate(static_cast<size_t>(NumBytes), 0);
+ constexpr size_t Alignment = 4;
+ // Source buffer offset must be multiple of 4 (18.4)
+ auto TmpSpace = m_UploadHeap.Allocate(static_cast<size_t>(NumBytes), Alignment);
memcpy(TmpSpace.CPUAddress, pData, static_cast<size_t>(NumBytes));
UpdateBufferRegion(pBuffVk, DstOffset, NumBytes, TmpSpace.vkBuffer, TmpSpace.AlignedOffset);
// The allocation will stay in the upload heap until the end of the frame at which point all upload
@@ -1161,7 +1163,19 @@ namespace Diligent
CopyInfo.RowCount = UpdateRegionHeight;
}
- CopyInfo.Stride = CopyInfo.RowSize;
+ const auto& DeviceLimits = m_pDevice.RawPtr<const RenderDeviceVkImpl>()->GetPhysicalDevice().GetProperties().limits;
+ CopyInfo.Stride = Align(CopyInfo.RowSize, static_cast<Uint32>(DeviceLimits.optimalBufferCopyRowPitchAlignment));
+ if (FmtAttribs.ComponentType == COMPONENT_TYPE_COMPRESSED)
+ {
+ // If the calling command’s VkImage parameter is a compressed image,
+ // bufferRowLength must be a multiple of the compressed texel block width
+ // In texels (not even in compressed blocks!)
+ CopyInfo.StrideInTexels = CopyInfo.Stride / Uint32{FmtAttribs.ComponentSize} * Uint32{FmtAttribs.BlockWidth};
+ }
+ else
+ {
+ CopyInfo.StrideInTexels = CopyInfo.Stride / (Uint32{FmtAttribs.ComponentSize} * Uint32{FmtAttribs.NumComponents});
+ }
CopyInfo.DepthStride = CopyInfo.RowCount * CopyInfo.Stride;
CopyInfo.MemorySize = UpdateRegionDepth * CopyInfo.DepthStride;
CopyInfo.Region = Region;
@@ -1183,7 +1197,9 @@ namespace Diligent
const auto UpdateRegionDepth = CopyInfo.Region.MaxZ - CopyInfo.Region.MinZ;
// For UpdateTextureRegion(), use UploadHeap, not dynamic heap
- static constexpr const size_t BufferOffsetAlignment = 4; // bufferOffset of VkBufferImageCopy must be a multiple of 4 (18.4)
+ const auto& DeviceLimits = m_pDevice.RawPtr<const RenderDeviceVkImpl>()->GetPhysicalDevice().GetProperties().limits;
+ // Source buffer offset must be multiple of 4 (18.4)
+ auto BufferOffsetAlignment = std::max(DeviceLimits.optimalBufferCopyOffsetAlignment, VkDeviceSize{4});
auto Allocation = m_UploadHeap.Allocate(CopyInfo.MemorySize, BufferOffsetAlignment);
// The allocation will stay in the upload heap until the end of the frame at which point all upload
// pages will be discarded
@@ -1212,6 +1228,7 @@ namespace Diligent
}
CopyBufferToTexture(Allocation.vkBuffer,
static_cast<Uint32>(Allocation.AlignedOffset),
+ CopyInfo.StrideInTexels,
CopyInfo.Region,
TextureVk,
MipLevel,
@@ -1220,6 +1237,7 @@ namespace Diligent
void DeviceContextVkImpl::CopyBufferToTexture(VkBuffer vkBuffer,
Uint32 BufferOffset,
+ Uint32 BufferRowStrideInTexels,
const Box& Region,
TextureVkImpl& TextureVk,
Uint32 MipLevel,
@@ -1238,7 +1256,7 @@ namespace Diligent
// bufferRowLength and bufferImageHeight specify the data in buffer memory as a subregion of a larger two- or
// three-dimensional image, and control the addressing calculations of data in buffer memory. If either of these
// values is zero, that aspect of the buffer memory is considered to be tightly packed according to the imageExtent (18.4).
- CopyRegion.bufferRowLength = 0;
+ CopyRegion.bufferRowLength = BufferRowStrideInTexels;
CopyRegion.bufferImageHeight = 0;
const auto& TexDesc = TextureVk.GetDesc();
@@ -1300,10 +1318,12 @@ namespace Diligent
{
const auto& TexDesc = TextureVk.GetDesc();
auto CopyInfo = GetBufferToTextureCopyInfo(TexDesc, MipLevel, MapRegion);
- auto Allocation = AllocateDynamicSpace(CopyInfo.MemorySize);
- VERIFY( (Allocation.Offset % 4) == 0, "Allocation offset must be at least 32-bit algined");
+ const auto& DeviceLimits = m_pDevice.RawPtr<RenderDeviceVkImpl>()->GetPhysicalDevice().GetProperties().limits;
+ // Source buffer offset must be multiple of 4 (18.4)
+ auto Alignment = std::max(DeviceLimits.optimalBufferCopyOffsetAlignment, VkDeviceSize{4});
+ auto Allocation = AllocateDynamicSpace(CopyInfo.MemorySize, static_cast<Uint32>(Alignment));
- MappedData.pData = reinterpret_cast<Uint8*>(Allocation.pDynamicMemMgr->GetCPUAddress()) + Allocation.Offset;
+ MappedData.pData = reinterpret_cast<Uint8*>(Allocation.pDynamicMemMgr->GetCPUAddress()) + Allocation.AlignedOffset;
MappedData.Stride = CopyInfo.Stride;
MappedData.DepthStride = CopyInfo.DepthStride;
@@ -1322,7 +1342,8 @@ namespace Diligent
{
auto& MappedTex = UploadSpaceIt->second;
CopyBufferToTexture(MappedTex.Allocation.pDynamicMemMgr->GetVkBuffer(),
- static_cast<Uint32>(MappedTex.Allocation.Offset),
+ static_cast<Uint32>(MappedTex.Allocation.AlignedOffset),
+ MappedTex.CopyInfo.StrideInTexels,
MappedTex.CopyInfo.Region,
TextureVk,
MipLevel,
@@ -1495,9 +1516,9 @@ namespace Diligent
BufferVk.SetAccessFlags(NewAccessFlags);
}
- VulkanDynamicAllocation DeviceContextVkImpl::AllocateDynamicSpace(Uint32 SizeInBytes)
+ VulkanDynamicAllocation DeviceContextVkImpl::AllocateDynamicSpace(Uint32 SizeInBytes, Uint32 Alignment)
{
- auto DynAlloc = m_DynamicHeap.Allocate(SizeInBytes, 0);
+ auto DynAlloc = m_DynamicHeap.Allocate(SizeInBytes, Alignment);
#ifdef DEVELOPMENT
DynAlloc.dvpFrameNumber = m_ContextFrameNumber;
#endif
diff --git a/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp
index 476766aa..b6ab45bf 100644
--- a/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/RenderDeviceVkImpl.cpp
@@ -323,7 +323,7 @@ void RenderDeviceVkImpl::FinishFrame(bool ReleaseAllResources)
// until the GPU is finished with the current frame
ProcessStaleResources(SubmittedCmdBuffNumber, SubmittedFenceValue, CompletedFenceValue);
- m_DynamicMemoryManager.ReleaseStaleAllocations(CompletedFenceValue);
+ m_DynamicMemoryManager.ReleaseStaleBlocks(CompletedFenceValue);
Atomics::AtomicIncrement(m_FrameNumber);
}
diff --git a/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp
index b2487bda..b95094f4 100644
--- a/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/TextureVkImpl.cpp
@@ -150,8 +150,10 @@ TextureVkImpl :: TextureVkImpl(IReferenceCounters* pRefCounters,
else
ImageMemoryFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
+ VERIFY( IsPowerOfTwo(MemReqs.alignment), "Alignment is not power of 2!");
m_MemoryAllocation = pRenderDeviceVk->AllocateMemory(MemReqs, ImageMemoryFlags);
- auto AlignedOffset = (m_MemoryAllocation.UnalignedOffset + (MemReqs.alignment-1)) & ~(MemReqs.alignment-1);
+ auto AlignedOffset = Align(m_MemoryAllocation.UnalignedOffset, MemReqs.alignment);
+ VERIFY_EXPR(m_MemoryAllocation.Size >= MemReqs.size + (AlignedOffset - m_MemoryAllocation.UnalignedOffset));
auto Memory = m_MemoryAllocation.Page->GetVkMemory();
auto err = LogicalDevice.BindImageMemory(m_VulkanImage, Memory, AlignedOffset);
CHECK_VK_ERROR_AND_THROW(err, "Failed to bind image memory");
@@ -268,12 +270,14 @@ TextureVkImpl :: TextureVkImpl(IReferenceCounters* pRefCounters,
VulkanUtilities::BufferWrapper StagingBuffer = LogicalDevice.CreateBuffer(VkStaginBuffCI, StagingBufferName.c_str());
VkMemoryRequirements StagingBufferMemReqs = LogicalDevice.GetBufferMemoryRequirements(StagingBuffer);
+ VERIFY( IsPowerOfTwo(StagingBufferMemReqs.alignment), "Alignment is not power of 2!");
// VK_MEMORY_PROPERTY_HOST_COHERENT_BIT bit specifies that the host cache management commands vkFlushMappedMemoryRanges
// and vkInvalidateMappedMemoryRanges are NOT needed to flush host writes to the device or make device writes visible
// to the host (10.2)
auto StagingMemoryAllocation = pRenderDeviceVk->AllocateMemory(StagingBufferMemReqs, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
auto StagingBufferMemory = StagingMemoryAllocation.Page->GetVkMemory();
- auto AlignedStagingMemOffset = (StagingMemoryAllocation.UnalignedOffset + (StagingBufferMemReqs.alignment-1)) & ~(StagingBufferMemReqs.alignment-1);
+ auto AlignedStagingMemOffset = Align(StagingMemoryAllocation.UnalignedOffset, StagingBufferMemReqs.alignment);
+ VERIFY_EXPR(StagingMemoryAllocation.Size >= StagingBufferMemReqs.size + (AlignedStagingMemOffset - StagingMemoryAllocation.UnalignedOffset));
auto *StagingData = reinterpret_cast<uint8_t*>(StagingMemoryAllocation.Page->GetCPUMemory());
VERIFY_EXPR(StagingData != nullptr);
diff --git a/Graphics/GraphicsEngineVulkan/src/VulkanDynamicHeap.cpp b/Graphics/GraphicsEngineVulkan/src/VulkanDynamicHeap.cpp
index 429c174c..e535dcad 100644
--- a/Graphics/GraphicsEngineVulkan/src/VulkanDynamicHeap.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/VulkanDynamicHeap.cpp
@@ -40,11 +40,11 @@ static VkDeviceSize GetDefaultAlignment(const VulkanUtilities::VulkanPhysicalDev
VulkanDynamicMemoryManager::VulkanDynamicMemoryManager(IMemoryAllocator& Allocator,
RenderDeviceVkImpl& DeviceVk,
Uint32 Size) :
+ TBase(Allocator, Size),
m_DeviceVk(DeviceVk),
- m_DefaultAlignment(GetDefaultAlignment(DeviceVk.GetPhysicalDevice())),
- m_AllocationStrategy(Allocator, Size)
+ m_DefaultAlignment(GetDefaultAlignment(DeviceVk.GetPhysicalDevice()))
{
- VERIFY( (Size & (MinAlignment-1)) == 0, "Heap size is not min aligned");
+ VERIFY( (Size & (MasterBlockAlignment-1)) == 0, "Heap size (", Size, " is not aligned by the master block alignment (", MasterBlockAlignment, ")");
VkBufferCreateInfo VkBuffCI = {};
VkBuffCI.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO;
VkBuffCI.pNext = nullptr;
@@ -114,24 +114,27 @@ void VulkanDynamicMemoryManager::Destroy()
VulkanDynamicMemoryManager::~VulkanDynamicMemoryManager()
{
VERIFY(m_BufferMemory == VK_NULL_HANDLE && m_VkBuffer == VK_NULL_HANDLE, "Vulkan resources must be explcitly released with Destroy()");
- auto Size = m_AllocationStrategy.GetSize();
+ auto Size = GetSize();
LOG_INFO_MESSAGE("Dynamic memory manager usage stats:\n"
" Total size: ", FormatMemorySize(Size, 2),
". Peak allocated size: ", FormatMemorySize(m_TotalPeakSize, 2, Size),
". Peak utilization: ", std::fixed, std::setprecision(1), static_cast<double>(m_TotalPeakSize) / static_cast<double>(std::max(Size, size_t{1})) * 100.0, '%' );
}
-VulkanDynamicMemoryManager::OffsetType VulkanDynamicMemoryManager::Allocate(OffsetType SizeInBytes)
+
+VulkanDynamicMemoryManager::MasterBlock VulkanDynamicMemoryManager::AllocateMasterBlock(OffsetType SizeInBytes, OffsetType Alignment)
{
- VERIFY( (SizeInBytes & (MinAlignment-1)) == 0, "Allocation size is not minimally aligned" );
-
- if (SizeInBytes > m_AllocationStrategy.GetSize())
+ if (Alignment == 0)
+ Alignment = MasterBlockAlignment;
+
+ if (SizeInBytes > GetSize())
{
- LOG_ERROR("Requested dynamic allocation size ", SizeInBytes, " exceeds maximum dynamic memory size ", m_AllocationStrategy.GetSize(), ". The app should increase dynamic heap size.");
- return AllocationStategy::InvalidOffset;
+ LOG_ERROR("Requested dynamic allocation size ", SizeInBytes, " exceeds maximum dynamic memory size ", GetSize(), ". The app should increase dynamic heap size.");
+ return MasterBlock{};
}
- auto Offset = m_AllocationStrategy.Allocate(SizeInBytes);
- if(Offset == AllocationStategy::InvalidOffset)
+
+ auto Block = TBase::AllocateMasterBlock(SizeInBytes, Alignment);
+ if (Block.UnalignedOffset == InvalidOffset)
{
// Allocation failed. Try to wait for GPU to finish pending frames to release some space
auto StartIdleTime = std::chrono::high_resolution_clock::now();
@@ -139,12 +142,12 @@ VulkanDynamicMemoryManager::OffsetType VulkanDynamicMemoryManager::Allocate(Offs
static constexpr const auto MaxIdleDuration = std::chrono::duration<double>{60.0 / 1000.0}; // 60 ms
std::chrono::duration<double> IdleDuration;
Uint32 SleepIterations = 0;
- while(Offset == AllocationStategy::InvalidOffset && IdleDuration < MaxIdleDuration)
+ while (Block.UnalignedOffset == InvalidOffset && IdleDuration < MaxIdleDuration)
{
auto LastCompletedFenceValue = m_DeviceVk.GetCompletedFenceValue();
- m_AllocationStrategy.ReleaseStaleAllocations(LastCompletedFenceValue);
- Offset = m_AllocationStrategy.Allocate(SizeInBytes);
- if (Offset == AllocationStategy::InvalidOffset)
+ ReleaseStaleBlocks(LastCompletedFenceValue);
+ Block = AllocateMasterBlock(SizeInBytes, Alignment);
+ if (Block.UnalignedOffset == InvalidOffset)
{
std::this_thread::sleep_for(SleepPeriod);
++SleepIterations;
@@ -154,14 +157,14 @@ VulkanDynamicMemoryManager::OffsetType VulkanDynamicMemoryManager::Allocate(Offs
IdleDuration = std::chrono::duration_cast<std::chrono::duration<double>>(CurrTime - StartIdleTime);
}
- if(Offset == AllocationStategy::InvalidOffset)
+ if (Block.UnalignedOffset == InvalidOffset)
{
// Last resort - idle GPU (there seems to be a driver bug: vkQueueWaitIdle() deadlocks and never returns)
//m_DeviceVk.IdleGPU(true);
//auto LastCompletedFenceValue = m_DeviceVk.GetCompletedFenceValue();
//m_AllocationStrategy.ReleaseStaleAllocations(LastCompletedFenceValue);
//Offset = m_AllocationStrategy.Allocate(SizeInBytes);
- if (Offset == AllocationStategy::InvalidOffset)
+ if (Block.UnalignedOffset == InvalidOffset)
{
LOG_ERROR_MESSAGE("Space in dynamic heap is exausted! After idling for ", std::fixed, std::setprecision(1), IdleDuration.count()*1000.0, " ms still no space is available. Increase the size of the heap by setting EngineVkAttribs::DynamicHeapSize to a greater value or optimize dynamic resource usage");
}
@@ -183,59 +186,71 @@ VulkanDynamicMemoryManager::OffsetType VulkanDynamicMemoryManager::Allocate(Offs
}
}
- if (Offset != AllocationStategy::InvalidOffset)
+ if (Block.UnalignedOffset != InvalidOffset)
{
- m_TotalPeakSize = std::max(m_TotalPeakSize, m_AllocationStrategy.GetUsedSize());
+ m_TotalPeakSize = std::max(m_TotalPeakSize, GetUsedSize());
}
- return Offset;
+
+ return Block;
}
VulkanDynamicAllocation VulkanDynamicHeap::Allocate(Uint32 SizeInBytes, Uint32 Alignment)
{
- if (Alignment == 0)
- Alignment = static_cast<Uint32>(m_DynamicMemMgr.m_DefaultAlignment);
-
- const Uint32 AlignmentMask = Alignment - 1;
- // Assert that it's a power of two.
- VERIFY_EXPR((AlignmentMask & Alignment) == 0);
+ VERIFY_EXPR(Alignment > 0);
+ VERIFY(IsPowerOfTwo(Alignment), "Alignment (", Alignment, ") must be power of 2");
- // Align the allocation
- Uint32 AlignedSize = (SizeInBytes + AlignmentMask) & ~AlignmentMask;
-
- auto Offset = InvalidOffset;
- if(AlignedSize > m_PagSize/2)
+ auto AlignedOffset = InvalidOffset;
+ OffsetType AlignedSize = 0;
+ if(SizeInBytes > m_MasterBlockSize/2)
{
- AlignedSize = (SizeInBytes + VulkanDynamicMemoryManager::MinAlignment) & ~(VulkanDynamicMemoryManager::MinAlignment-1);
// Allocate directly from the memory manager
- Offset = m_DynamicMemMgr.Allocate(AlignedSize);
- m_Allocations.emplace_back(Offset, AlignedSize);
+ auto MasterBlock = m_DynamicMemMgr.AllocateMasterBlock(SizeInBytes, Alignment);
+ if (MasterBlock.UnalignedOffset != InvalidOffset)
+ {
+ AlignedOffset = Align(MasterBlock.UnalignedOffset, size_t{Alignment});
+ AlignedSize = MasterBlock.Size;
+ VERIFY_EXPR(MasterBlock.Size >= SizeInBytes + (AlignedOffset - MasterBlock.UnalignedOffset));
+ m_MasterBlocks.emplace_back(MasterBlock);
+ }
}
else
{
- if(m_CurrOffset == InvalidOffset || AlignedSize > m_AvailableSize)
+ if (m_CurrOffset == InvalidOffset || SizeInBytes + (Align(m_CurrOffset, size_t{Alignment}) - m_CurrOffset) > m_AvailableSize)
{
- m_CurrOffset = m_DynamicMemMgr.Allocate(m_PagSize);
- m_Allocations.emplace_back(m_CurrOffset, m_PagSize);
- m_AvailableSize = m_PagSize;
+ auto MasterBlock = m_DynamicMemMgr.AllocateMasterBlock(m_MasterBlockSize, 0);
+ if (MasterBlock.UnalignedOffset != InvalidOffset)
+ {
+ m_CurrOffset = MasterBlock.UnalignedOffset;
+ m_MasterBlocks.emplace_back(MasterBlock);
+ m_AvailableSize = m_MasterBlockSize;
+ }
}
- if(m_CurrOffset != InvalidOffset)
+
+ if (m_CurrOffset != InvalidOffset)
{
- Offset = m_CurrOffset;
- m_AvailableSize -= AlignedSize;
- m_CurrOffset += AlignedSize;
+ AlignedOffset = Align(m_CurrOffset, size_t{Alignment});
+ AlignedSize = SizeInBytes + (AlignedOffset - m_CurrOffset);
+ if (AlignedSize <= m_AvailableSize)
+ {
+ m_AvailableSize -= static_cast<Uint32>(AlignedSize);
+ m_CurrOffset += static_cast<Uint32>(AlignedSize);
+ }
+ else
+ AlignedOffset = InvalidOffset;
}
}
// Every device context uses its own dynamic heap, so there is no need to lock
- if(Offset != InvalidOffset)
+ if(AlignedOffset != InvalidOffset)
{
- m_CurrAllocatedSize += AlignedSize;
+ m_CurrAllocatedSize += static_cast<Uint32>(AlignedSize);
m_CurrUsedSize += SizeInBytes;
m_PeakAllocatedSize = std::max(m_PeakAllocatedSize, m_CurrAllocatedSize);
m_PeakUsedSize = std::max(m_PeakUsedSize, m_CurrUsedSize);
-
- return VulkanDynamicAllocation{ m_DynamicMemMgr, Offset, SizeInBytes };
+
+ VERIFY_EXPR((AlignedOffset & (Alignment-1)) == 0);
+ return VulkanDynamicAllocation{ m_DynamicMemMgr, AlignedOffset, SizeInBytes };
}
else
return VulkanDynamicAllocation{};
@@ -243,8 +258,8 @@ VulkanDynamicAllocation VulkanDynamicHeap::Allocate(Uint32 SizeInBytes, Uint32 A
void VulkanDynamicHeap::FinishFrame(Uint64 FenceValue)
{
- m_DynamicMemMgr.DiscardAllocations(m_Allocations, FenceValue);
- m_Allocations.clear();
+ m_DynamicMemMgr.DiscardMasterBlocks(m_MasterBlocks, FenceValue);
+ m_MasterBlocks.clear();
m_CurrOffset = InvalidOffset;
m_AvailableSize = 0;
diff --git a/Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp b/Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp
index aba420e8..a41a010f 100644
--- a/Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/VulkanUploadHeap.cpp
@@ -81,12 +81,7 @@ VulkanUploadHeap::UploadPageInfo VulkanUploadHeap::CreateNewPage(VkDeviceSize Si
VulkanUploadAllocation VulkanUploadHeap::Allocate(size_t SizeInBytes, size_t Alignment)
{
- VERIFY((Alignment & (Alignment-1)) == 0, "Alignment (", Alignment, ") must be power of two");
- if (Alignment == 0)
- {
- static constexpr const size_t MinimumAlignment = 4;
- Alignment = MinimumAlignment;
- }
+ VERIFY(IsPowerOfTwo(Alignment), "Alignment (", Alignment, ") must be power of two");
VulkanUploadAllocation Allocation;
if(SizeInBytes >= m_PageSize/2)
@@ -103,7 +98,7 @@ VulkanUploadAllocation VulkanUploadHeap::Allocate(size_t SizeInBytes, size_t Ali
}
else
{
- auto AlignmentOffset = ((m_CurrPage.CurrOffset + (Alignment-1)) & ~(Alignment-1)) - m_CurrPage.CurrOffset;
+ auto AlignmentOffset = Align(m_CurrPage.CurrOffset, Alignment) - m_CurrPage.CurrOffset;
if(m_CurrPage.AvailableSize < SizeInBytes + AlignmentOffset)
{
// Allocate new page
diff --git a/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanMemoryManager.cpp b/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanMemoryManager.cpp
index b4660819..973b1add 100644
--- a/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanMemoryManager.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/VulkanUtilities/VulkanMemoryManager.cpp
@@ -32,7 +32,7 @@ VulkanMemoryAllocation::~VulkanMemoryAllocation()
{
if (Page != nullptr)
{
- Page->Free(*this);
+ Page->Free(std::move(*this));
}
}
@@ -75,13 +75,16 @@ VulkanMemoryPage::~VulkanMemoryPage()
VERIFY(IsEmpty(), "Destroying a page with not all allocations released");
}
-VulkanMemoryAllocation VulkanMemoryPage::Allocate(VkDeviceSize size)
+VulkanMemoryAllocation VulkanMemoryPage::Allocate(VkDeviceSize size, VkDeviceSize alignment)
{
std::lock_guard<std::mutex> Lock(m_Mutex);
- auto Offset = m_AllocationMgr.Allocate(size);
- if (Offset != Diligent::VariableSizeAllocationsManager::InvalidOffset)
+ auto Allocation = m_AllocationMgr.Allocate(size, alignment);
+ if (Allocation.UnalignedOffset != Diligent::VariableSizeAllocationsManager::InvalidOffset)
{
- return VulkanMemoryAllocation{this, Offset, size};
+ // Offset may not necessarily be aligned, but the allocation is guaranteed to be large enough
+ // to accomodate requested alignment
+ VERIFY_EXPR( Diligent::Align(Allocation.UnalignedOffset, alignment) - Allocation.UnalignedOffset + size <= Allocation.Size );
+ return VulkanMemoryAllocation{this, Allocation.UnalignedOffset, Allocation.Size};
}
else
{
@@ -89,11 +92,12 @@ VulkanMemoryAllocation VulkanMemoryPage::Allocate(VkDeviceSize size)
}
}
-void VulkanMemoryPage::Free(VulkanMemoryAllocation& Allocation)
+void VulkanMemoryPage::Free(VulkanMemoryAllocation&& Allocation)
{
m_ParentMemoryMgr.OnFreeAllocation(Allocation.Size, m_CPUMemory != nullptr);
std::lock_guard<std::mutex> Lock(m_Mutex);
m_AllocationMgr.Free(Allocation.UnalignedOffset, Allocation.Size);
+ Allocation = VulkanMemoryAllocation{};
}
VulkanMemoryAllocation VulkanMemoryManager::Allocate(const VkMemoryRequirements& MemReqs, VkMemoryPropertyFlags MemoryProps)
@@ -130,14 +134,13 @@ VulkanMemoryAllocation VulkanMemoryManager::Allocate(const VkMemoryRequirements&
VulkanMemoryAllocation VulkanMemoryManager::Allocate(VkDeviceSize Size, VkDeviceSize Alignment, uint32_t MemoryTypeIndex, bool HostVisible)
{
- Size += Alignment;
VulkanMemoryAllocation Allocation;
std::lock_guard<std::mutex> Lock(m_PagesMtx);
auto range = m_Pages.equal_range(MemoryTypeIndex);
for(auto page_it = range.first; page_it != range.second; ++page_it)
{
- Allocation = page_it->second.Allocate(Size);
+ Allocation = page_it->second.Allocate(Size, Alignment);
if (Allocation.Page != nullptr)
break;
}
@@ -157,11 +160,16 @@ VulkanMemoryAllocation VulkanMemoryManager::Allocate(VkDeviceSize Size, VkDevice
" page. (", Diligent::FormatMemorySize(PageSize, 2), ", type idx: ", MemoryTypeIndex,
"). Current allocated size: ", Diligent::FormatMemorySize(m_CurrAllocatedSize[stat_ind], 2));
OnNewPageCreated(it->second);
- Allocation = it->second.Allocate(Size);
- VERIFY(Allocation.Page != nullptr, "Failed to allocate new memory page");
+ Allocation = it->second.Allocate(Size, Alignment);
+ DEV_CHECK_ERR(Allocation.Page != nullptr, "Failed to allocate new memory page");
}
- m_CurrUsedSize[stat_ind].fetch_add(Size);
+ if (Allocation.Page != nullptr)
+ {
+ VERIFY_EXPR(Size + Diligent::Align(Allocation.UnalignedOffset, Alignment) - Allocation.UnalignedOffset <= Allocation.Size);
+ }
+
+ m_CurrUsedSize[stat_ind].fetch_add(Allocation.Size);
m_PeakUsedSize[stat_ind] = std::max(m_PeakUsedSize[stat_ind], static_cast<VkDeviceSize>(m_CurrUsedSize[stat_ind].load()));
return Allocation;