Implemented queries in Vulkan (closed https://github.com/DiligentGraphics/DiligentCore/issues/25)

9 files changed, 602 insertions, 16 deletions

diff --git a/Graphics/GraphicsEngineVulkan/CMakeLists.txt b/Graphics/GraphicsEngineVulkan/CMakeLists.txt
index df141edb..bd670456 100644
--- a/Graphics/GraphicsEngineVulkan/CMakeLists.txt
+++ b/Graphics/GraphicsEngineVulkan/CMakeLists.txt
@@ -17,6 +17,7 @@ set(INCLUDE
     include/pch.h
     include/PipelineLayout.h
     include/PipelineStateVkImpl.h
+    include/QueryManagerVk.h
     include/QueryVkImpl.h
     include/RenderDeviceVkImpl.h
     include/RenderPassCache.h
@@ -81,6 +82,7 @@ set(SRC
     src/GenerateMipsVkHelper.cpp
     src/PipelineLayout.cpp
     src/PipelineStateVkImpl.cpp
+    src/QueryManagerVk.cpp
     src/QueryVkImpl.cpp
     src/RenderDeviceVkImpl.cpp
     src/RenderPassCache.cpp
diff --git a/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h b/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h
index 48a3a41b..f7939c08 100644
--- a/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h
+++ b/Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.h
@@ -47,6 +47,8 @@
 #include "QueryVkImpl.h"
 #include "HashUtils.h"
 #include "ManagedVulkanObject.h"
+#include "QueryManagerVk.h"
+
 
 namespace Diligent
 {
@@ -317,6 +319,7 @@ public:
     Int64 GetContextFrameNumber() const { return m_ContextFrameNumber; }
 
     GenerateMipsVkHelper& GetGenerateMipsHelper() { return *m_GenerateMipsHelper; }
+    QueryManagerVk&       GetQueryManager() { return m_QueryMgr; }
 
 private:
     void               TransitionRenderTargets(RESOURCE_STATE_TRANSITION_MODE StateTransitionMode);
@@ -468,6 +471,9 @@ private:
 
     // In Vulkan we can't bind null vertex buffer, so we have to create a dummy VB
     RefCntAutoPtr<BufferVkImpl> m_DummyVB;
+
+    QueryManagerVk m_QueryMgr;
+    Int32          m_ActiveQueriesCounter = 0;
 };
 
 } // namespace Diligent
diff --git a/Graphics/GraphicsEngineVulkan/include/QueryManagerVk.h b/Graphics/GraphicsEngineVulkan/include/QueryManagerVk.h
new file mode 100644
index 00000000..6df7b82c
--- /dev/null
+++ b/Graphics/GraphicsEngineVulkan/include/QueryManagerVk.h
@@ -0,0 +1,92 @@
+/*
+ *  Copyright 2019-2020 Diligent Graphics LLC
+ *  Copyright 2015-2019 Egor Yusov
+ *  
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  In no event and under no legal theory, whether in tort (including negligence), 
+ *  contract, or otherwise, unless required by applicable law (such as deliberate 
+ *  and grossly negligent acts) or agreed to in writing, shall any Contributor be
+ *  liable for any damages, including any direct, indirect, special, incidental, 
+ *  or consequential damages of any character arising as a result of this License or 
+ *  out of the use or inability to use the software (including but not limited to damages 
+ *  for loss of goodwill, work stoppage, computer failure or malfunction, or any and 
+ *  all other commercial damages or losses), even if such Contributor has been advised 
+ *  of the possibility of such damages.
+ */
+
+#pragma once
+
+#include <mutex>
+#include <array>
+#include <deque>
+#include <vector>
+
+#include "Query.h"
+#include "VulkanUtilities/VulkanLogicalDevice.h"
+#include "VulkanUtilities/VulkanPhysicalDevice.h"
+#include "VulkanUtilities/VulkanObjectWrappers.h"
+#include "VulkanUtilities/VulkanCommandBuffer.h"
+
+namespace Diligent
+{
+
+class QueryManagerVk
+{
+public:
+    QueryManagerVk(const VulkanUtilities::VulkanLogicalDevice&  LogicalDevice,
+                   const VulkanUtilities::VulkanPhysicalDevice& PhysicalDevice,
+                   const Uint32                                 QueryHeapSizes[]);
+    ~QueryManagerVk();
+
+    // clang-format off
+    QueryManagerVk             (const QueryManagerVk&)  = delete;
+    QueryManagerVk             (      QueryManagerVk&&) = delete;
+    QueryManagerVk& operator = (const QueryManagerVk&)  = delete;
+    QueryManagerVk& operator = (      QueryManagerVk&&) = delete;
+    // clang-format on
+
+    static constexpr Uint32 InvalidIndex = static_cast<Uint32>(-1);
+
+    Uint32 AllocateQuery(QUERY_TYPE Type);
+    void   DiscardQuery(QUERY_TYPE Type, Uint32 Index);
+
+    VkQueryPool GetQueryPool(QUERY_TYPE Type)
+    {
+        return m_Heaps[Type].vkQueryPool;
+    }
+
+    Uint64 GetCounterFrequency() const
+    {
+        return m_CounterFrequency;
+    }
+
+    void ResetStaleQueries(VulkanUtilities::VulkanCommandBuffer& CmdBuff);
+
+private:
+    struct QueryHeapInfo
+    {
+        VulkanUtilities::QueryPoolWrapper vkQueryPool;
+
+        std::deque<Uint32> AvailableQueries;
+        std::deque<Uint32> StaleQueries;
+        Uint32             PoolSize = 0;
+    };
+
+    std::mutex                                      m_HeapMutex;
+    std::array<QueryHeapInfo, QUERY_TYPE_NUM_TYPES> m_Heaps;
+
+    Uint64 m_CounterFrequency = 0;
+};
+
+} // namespace Diligent
diff --git a/Graphics/GraphicsEngineVulkan/include/QueryVkImpl.h b/Graphics/GraphicsEngineVulkan/include/QueryVkImpl.h
index 3574cdfd..aa88d71b 100644
--- a/Graphics/GraphicsEngineVulkan/include/QueryVkImpl.h
+++ b/Graphics/GraphicsEngineVulkan/include/QueryVkImpl.h
@@ -33,6 +33,7 @@
 #include "QueryVk.h"
 #include "QueryBase.h"
 #include "RenderDeviceVkImpl.h"
+#include "QueryManagerVk.h"
 
 namespace Diligent
 {
@@ -55,7 +56,19 @@ public:
 
     virtual bool GetData(void* pData, Uint32 DataSize) override final;
 
+    Uint32 GetQueryPoolIndex() const
+    {
+        return m_QueryPoolIndex;
+    }
+
+    bool OnEndQuery(IDeviceContext* pContext);
+    bool OnBeginQuery(IDeviceContext* pContext);
+
 private:
+    bool AllocateQuery();
+
+    Uint32 m_QueryPoolIndex     = QueryManagerVk::InvalidIndex;
+    Uint64 m_QueryEndFenceValue = 0;
 };
 
 } // namespace Diligent
diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanCommandBuffer.h b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanCommandBuffer.h
index b1497ef1..811a1671 100644
--- a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanCommandBuffer.h
+++ b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanCommandBuffer.h
@@ -199,6 +199,13 @@ public:
         m_State.Framebuffer       = VK_NULL_HANDLE;
         m_State.FramebufferWidth  = 0;
         m_State.FramebufferHeight = 0;
+        if (m_State.InsidePassQueries != 0)
+        {
+            LOG_ERROR_MESSAGE("There are outstanding queries that have been started inside the render pass, but have "
+                              "not been ended. Vulkan requires that a query must either begin and end inside the same "
+                              "subpass of a render pass instance, or must both begin and end outside of a render pass "
+                              "instance (i.e. contain entire render pass instances). (17.2)");
+        }
     }
 
     __forceinline void EndCommandBuffer()
@@ -443,26 +450,58 @@ public:
 
     __forceinline void BeginQuery(VkQueryPool         queryPool,
                                   uint32_t            query,
-                                  VkQueryControlFlags flags)
+                                  VkQueryControlFlags flags,
+                                  uint32_t            queryFlag)
     {
+        if ((m_State.InsidePassQueries | m_State.OutsidePassQueries) & queryFlag)
+        {
+            LOG_ERROR_MESSAGE("Another query of the same type is already active. "
+                              "Overlapping queries do not work in Vulkan. The command will be ignored.");
+            return;
+        }
+
         // A query must either begin and end inside the same subpass of a render
         // pass instance, or must both begin and end outside of a render pass instance
         // (i.e. contain entire render pass instances) (17.2).
         VERIFY_EXPR(m_VkCmdBuffer != VK_NULL_HANDLE);
         vkCmdBeginQuery(m_VkCmdBuffer, queryPool, query, flags);
+        if (m_State.RenderPass != VK_NULL_HANDLE)
+            m_State.InsidePassQueries |= queryFlag;
+        else
+            m_State.OutsidePassQueries |= queryFlag;
     }
 
     __forceinline void EndQuery(VkQueryPool queryPool,
-                                uint32_t    query)
+                                uint32_t    query,
+                                uint32_t    queryFlag)
     {
         VERIFY_EXPR(m_VkCmdBuffer != VK_NULL_HANDLE);
         vkCmdEndQuery(m_VkCmdBuffer, queryPool, query);
+        if (m_State.RenderPass != VK_NULL_HANDLE)
+        {
+            VERIFY((m_State.InsidePassQueries & queryFlag) != 0, "No active inside-pass queries found.");
+            m_State.InsidePassQueries &= ~queryFlag;
+        }
+        else
+        {
+            VERIFY((m_State.OutsidePassQueries & queryFlag) != 0, "No active outside-pass queries found.");
+            m_State.OutsidePassQueries &= ~queryFlag;
+        }
+    }
+
+    __forceinline void WriteTimestamp(VkPipelineStageFlagBits pipelineStage,
+                                      VkQueryPool             queryPool,
+                                      uint32_t                query)
+    {
+        VERIFY_EXPR(m_VkCmdBuffer != VK_NULL_HANDLE);
+        vkCmdWriteTimestamp(m_VkCmdBuffer, pipelineStage, queryPool, query);
     }
 
     __forceinline void ResetQueryPool(VkQueryPool queryPool,
                                       uint32_t    firstQuery,
                                       uint32_t    queryCount)
     {
+        VERIFY_EXPR(m_VkCmdBuffer != VK_NULL_HANDLE);
         if (m_State.RenderPass != VK_NULL_HANDLE)
         {
             // Query pool reset must be performed outside of render pass (17.2).
@@ -499,15 +538,17 @@ public:
 
     struct StateCache
     {
-        VkRenderPass  RenderPass        = VK_NULL_HANDLE;
-        VkFramebuffer Framebuffer       = VK_NULL_HANDLE;
-        VkPipeline    GraphicsPipeline  = VK_NULL_HANDLE;
-        VkPipeline    ComputePipeline   = VK_NULL_HANDLE;
-        VkBuffer      IndexBuffer       = VK_NULL_HANDLE;
-        VkDeviceSize  IndexBufferOffset = 0;
-        VkIndexType   IndexType         = VK_INDEX_TYPE_MAX_ENUM;
-        uint32_t      FramebufferWidth  = 0;
-        uint32_t      FramebufferHeight = 0;
+        VkRenderPass  RenderPass         = VK_NULL_HANDLE;
+        VkFramebuffer Framebuffer        = VK_NULL_HANDLE;
+        VkPipeline    GraphicsPipeline   = VK_NULL_HANDLE;
+        VkPipeline    ComputePipeline    = VK_NULL_HANDLE;
+        VkBuffer      IndexBuffer        = VK_NULL_HANDLE;
+        VkDeviceSize  IndexBufferOffset  = 0;
+        VkIndexType   IndexType          = VK_INDEX_TYPE_MAX_ENUM;
+        uint32_t      FramebufferWidth   = 0;
+        uint32_t      FramebufferHeight  = 0;
+        uint32_t      InsidePassQueries  = 0;
+        uint32_t      OutsidePassQueries = 0;
     };
 
     const StateCache& GetState() const { return m_State; }
diff --git a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
index d6e9b826..3bdc41a1 100644
--- a/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/DeviceContextVkImpl.cpp
@@ -96,7 +96,8 @@ DeviceContextVkImpl::DeviceContextVkImpl(IReferenceCounters*                   p
         pDeviceVkImpl->GetDynamicDescriptorPool(),
         GetContextObjectName("Dynamic descriptor set allocator", bIsDeferred, ContextId),
     },
-    m_GenerateMipsHelper{std::move(GenerateMipsHelper)}
+    m_GenerateMipsHelper{std::move(GenerateMipsHelper)},
+    m_QueryMgr{pDeviceVkImpl->GetLogicalDevice(), pDeviceVkImpl->GetPhysicalDevice(), EngineCI.QueryPoolSizes}
 // clang-format on
 {
     m_GenerateMipsHelper->CreateSRB(&m_GenerateMipsSRB);
@@ -227,7 +228,8 @@ void DeviceContextVkImpl::SetPipelineState(IPipelineState* pPipelineState)
         return;
 
     // Never flush deferred context!
-    if (!m_bIsDeferred && m_State.NumCommands >= m_NumCommandsToFlush)
+    // A query must begin and end in the same command buffer (17.2)
+    if (!m_bIsDeferred && m_State.NumCommands >= m_NumCommandsToFlush && m_ActiveQueriesCounter == 0)
     {
         Flush();
     }
@@ -824,6 +826,13 @@ void DeviceContextVkImpl::FinishFrame()
         }
     }
 
+    if (m_ActiveQueriesCounter > 0)
+    {
+        LOG_ERROR_MESSAGE("There are ", m_ActiveQueriesCounter,
+                          " active queries in the device context when finishing the frame. "
+                          "All queries must be ended before the frame is finished.");
+    }
+
     if (!m_MappedTextures.empty())
         LOG_ERROR_MESSAGE("There are mapped textures in the device context when finishing the frame. All dynamic resources must be used in the same frame in which they are mapped.");
 
@@ -857,6 +866,12 @@ void DeviceContextVkImpl::Flush()
         return;
     }
 
+    if (m_ActiveQueriesCounter > 0)
+    {
+        LOG_ERROR_MESSAGE("Flushing device context that has ", m_ActiveQueriesCounter,
+                          " active queries. Vulkan requires that queries are begun and ended in the same command buffer");
+    }
+
     VkSubmitInfo SubmitInfo = {};
 
     SubmitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
@@ -865,6 +880,8 @@ void DeviceContextVkImpl::Flush()
     auto vkCmdBuff = m_CommandBuffer.GetVkCmdBuffer();
     if (vkCmdBuff != VK_NULL_HANDLE)
     {
+        m_QueryMgr.ResetStaleQueries(m_CommandBuffer);
+
         if (m_State.NumCommands != 0)
         {
             if (m_CommandBuffer.GetState().RenderPass != VK_NULL_HANDLE)
@@ -1995,7 +2012,38 @@ void DeviceContextVkImpl::BeginQuery(IQuery* pQuery)
     if (!TDeviceContextBase::BeginQuery(pQuery, 0))
         return;
 
-    auto* pQueryVkImpl = ValidatedCast<QueryVkImpl>(pQuery);
+    auto*      pQueryVkImpl = ValidatedCast<QueryVkImpl>(pQuery);
+    const auto QueryType    = pQueryVkImpl->GetDesc().Type;
+    auto       Idx          = pQueryVkImpl->GetQueryPoolIndex();
+
+    EnsureVkCmdBuffer();
+    if (QueryType == QUERY_TYPE_TIMESTAMP)
+    {
+        LOG_ERROR_MESSAGE("BeginQuery() is disabled for timestamp queries");
+    }
+    else
+    {
+        const auto& CmdBuffState = m_CommandBuffer.GetState();
+        if ((CmdBuffState.InsidePassQueries | CmdBuffState.OutsidePassQueries) & (1u << QueryType))
+        {
+            LOG_ERROR_MESSAGE("Another query of type ", GetQueryTypeString(QueryType),
+                              " is currently active. Overlapping queries do not work in Vulkan. "
+                              "End the first query before beginning another one.");
+            return;
+        }
+
+        // A query must either begin and end inside the same subpass of a render pass instance, or must
+        // both begin and end outside of a render pass instance (i.e. contain entire render pass instances). (17.2)
+
+        ++m_ActiveQueriesCounter;
+        m_CommandBuffer.BeginQuery(m_QueryMgr.GetQueryPool(QueryType),
+                                   Idx,
+                                   // If flags does not contain VK_QUERY_CONTROL_PRECISE_BIT an implementation
+                                   // may generate any non-zero result value for the query if the count of
+                                   // passing samples is non-zero (17.3).
+                                   QueryType == QUERY_TYPE_OCCLUSION ? VK_QUERY_CONTROL_PRECISE_BIT : 0,
+                                   1u << QueryType);
+    }
 }
 
 void DeviceContextVkImpl::EndQuery(IQuery* pQuery)
@@ -2003,7 +2051,42 @@ void DeviceContextVkImpl::EndQuery(IQuery* pQuery)
     if (!TDeviceContextBase::EndQuery(pQuery, 0))
         return;
 
-    auto* pQueryVkImpl = ValidatedCast<QueryVkImpl>(pQuery);
+    auto*      pQueryVkImpl = ValidatedCast<QueryVkImpl>(pQuery);
+    const auto QueryType    = pQueryVkImpl->GetDesc().Type;
+    auto       vkQueryPool  = m_QueryMgr.GetQueryPool(QueryType);
+    auto       Idx          = pQueryVkImpl->GetQueryPoolIndex();
+
+    EnsureVkCmdBuffer();
+    if (QueryType == QUERY_TYPE_TIMESTAMP)
+    {
+        m_CommandBuffer.WriteTimestamp(VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, vkQueryPool, Idx);
+    }
+    else
+    {
+        VERIFY(m_ActiveQueriesCounter > 0, "Active query counter is 0 which means there was a mismatch between BeginQuery() / EndQuery() calls");
+
+        // A query must either begin and end inside the same subpass of a render pass instance, or must
+        // both begin and end outside of a render pass instance (i.e. contain entire render pass instances). (17.2)
+        const auto& CmdBuffState = m_CommandBuffer.GetState();
+        VERIFY((CmdBuffState.InsidePassQueries | CmdBuffState.OutsidePassQueries) & (1u << QueryType),
+               "No query flag is set which indicates there was no matching BeginQuery call or there was an error while beginning the query.");
+        if (CmdBuffState.OutsidePassQueries & (1 << QueryType))
+        {
+            if (m_CommandBuffer.GetState().RenderPass)
+                m_CommandBuffer.EndRenderPass();
+        }
+        else
+        {
+            if (!m_CommandBuffer.GetState().RenderPass)
+                LOG_ERROR_MESSAGE("The query was started inside render pass, but is being ended oustside of render pass. "
+                                  "Vulkan requires that a query must either begin and end inside the same "
+                                  "subpass of a render pass instance, or must both begin and end outside of a render pass "
+                                  "instance (i.e. contain entire render pass instances). (17.2)");
+        }
+
+        --m_ActiveQueriesCounter;
+        m_CommandBuffer.EndQuery(vkQueryPool, Idx, 1u << QueryType);
+    }
 }
 
 
diff --git a/Graphics/GraphicsEngineVulkan/src/EngineFactoryVk.cpp b/Graphics/GraphicsEngineVulkan/src/EngineFactoryVk.cpp
index c71b94e3..676d8ca3 100644
--- a/Graphics/GraphicsEngineVulkan/src/EngineFactoryVk.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/EngineFactoryVk.cpp
@@ -193,6 +193,9 @@ void EngineFactoryVkImpl::CreateDeviceAndContextsVk(const EngineVkCreateInfo& _E
         ENABLE_FEATURE(shaderStorageImageExtendedFormats)
 #undef ENABLE_FEATURE
 
+        DeviceFeatures.pipelineStatisticsQuery = VK_TRUE;
+        DeviceFeatures.occlusionQueryPrecise   = VK_TRUE;
+
         DeviceCreateInfo.pEnabledFeatures = &DeviceFeatures; // NULL or a pointer to a VkPhysicalDeviceFeatures structure that contains
                                                              // boolean indicators of all the features to be enabled.
 
diff --git a/Graphics/GraphicsEngineVulkan/src/QueryManagerVk.cpp b/Graphics/GraphicsEngineVulkan/src/QueryManagerVk.cpp
new file mode 100644
index 00000000..33231272
--- /dev/null
+++ b/Graphics/GraphicsEngineVulkan/src/QueryManagerVk.cpp
@@ -0,0 +1,188 @@
+/*
+ *  Copyright 2019-2020 Diligent Graphics LLC
+ *  Copyright 2015-2019 Egor Yusov
+ *  
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  In no event and under no legal theory, whether in tort (including negligence), 
+ *  contract, or otherwise, unless required by applicable law (such as deliberate 
+ *  and grossly negligent acts) or agreed to in writing, shall any Contributor be
+ *  liable for any damages, including any direct, indirect, special, incidental, 
+ *  or consequential damages of any character arising as a result of this License or 
+ *  out of the use or inability to use the software (including but not limited to damages 
+ *  for loss of goodwill, work stoppage, computer failure or malfunction, or any and 
+ *  all other commercial damages or losses), even if such Contributor has been advised 
+ *  of the possibility of such damages.
+ */
+
+#include "pch.h"
+#include "QueryManagerVk.h"
+#include "RenderDeviceVkImpl.h"
+#include "GraphicsAccessories.h"
+#include "VulkanUtilities/VulkanCommandBuffer.h"
+
+namespace Diligent
+{
+
+QueryManagerVk::QueryManagerVk(const VulkanUtilities::VulkanLogicalDevice&  LogicalDevice,
+                               const VulkanUtilities::VulkanPhysicalDevice& PhysicalDevice,
+                               const Uint32                                 QueryHeapSizes[])
+{
+    auto timestampPeriod = PhysicalDevice.GetProperties().limits.timestampPeriod;
+    m_CounterFrequency   = static_cast<Uint64>(1000000000.0 / timestampPeriod);
+
+    //Uint32 ResolveBufferOffset = 0;
+    for (Uint32 QueryType = QUERY_TYPE_UNDEFINED + 1; QueryType < QUERY_TYPE_NUM_TYPES; ++QueryType)
+    {
+        // clang-format off
+        static_assert(QUERY_TYPE_OCCLUSION          == 1, "Unexpected value of QUERY_TYPE_OCCLUSION. EngineVkCreateInfo::QueryPoolSizes must be updated");
+        static_assert(QUERY_TYPE_BINARY_OCCLUSION   == 2, "Unexpected value of QUERY_TYPE_BINARY_OCCLUSION. EngineVkCreateInfo::QueryPoolSizes must be updated");
+        static_assert(QUERY_TYPE_TIMESTAMP          == 3, "Unexpected value of QUERY_TYPE_TIMESTAMP. EngineVkCreateInfo::QueryPoolSizes must be updated");
+        static_assert(QUERY_TYPE_PIPELINE_STATISTICS== 4, "Unexpected value of QUERY_TYPE_PIPELINE_STATISTICS. EngineVkCreateInfo::QueryPoolSizes must be updated");
+        static_assert(QUERY_TYPE_NUM_TYPES          == 5, "Unexpected value of QUERY_TYPE_NUM_TYPES. EngineVkCreateInfo::QueryPoolSizes must be updated");
+        // clang-format on
+
+        auto& HeapInfo    = m_Heaps[QueryType];
+        HeapInfo.PoolSize = QueryHeapSizes[QueryType];
+
+        VkQueryPoolCreateInfo QueryPoolCI = {};
+
+        QueryPoolCI.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO;
+        QueryPoolCI.pNext = nullptr;
+        QueryPoolCI.flags = 0;
+        switch (QueryType)
+        {
+            case QUERY_TYPE_OCCLUSION:
+            case QUERY_TYPE_BINARY_OCCLUSION:
+                QueryPoolCI.queryType = VK_QUERY_TYPE_OCCLUSION;
+                break;
+
+            case QUERY_TYPE_TIMESTAMP:
+                QueryPoolCI.queryType = VK_QUERY_TYPE_TIMESTAMP;
+                break;
+
+            case QUERY_TYPE_PIPELINE_STATISTICS:
+            {
+                QueryPoolCI.queryType = VK_QUERY_TYPE_PIPELINE_STATISTICS;
+                QueryPoolCI.pipelineStatistics =
+                    VK_QUERY_PIPELINE_STATISTIC_INPUT_ASSEMBLY_VERTICES_BIT |
+                    VK_QUERY_PIPELINE_STATISTIC_INPUT_ASSEMBLY_PRIMITIVES_BIT |
+                    VK_QUERY_PIPELINE_STATISTIC_VERTEX_SHADER_INVOCATIONS_BIT |
+                    VK_QUERY_PIPELINE_STATISTIC_CLIPPING_INVOCATIONS_BIT |
+                    VK_QUERY_PIPELINE_STATISTIC_CLIPPING_PRIMITIVES_BIT |
+                    VK_QUERY_PIPELINE_STATISTIC_FRAGMENT_SHADER_INVOCATIONS_BIT |
+                    VK_QUERY_PIPELINE_STATISTIC_COMPUTE_SHADER_INVOCATIONS_BIT;
+
+                const auto EnabledShaderStages = LogicalDevice.GetEnabledGraphicsShaderStages();
+                if (EnabledShaderStages & VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT)
+                {
+                    QueryPoolCI.pipelineStatistics |=
+                        VK_QUERY_PIPELINE_STATISTIC_GEOMETRY_SHADER_INVOCATIONS_BIT |
+                        VK_QUERY_PIPELINE_STATISTIC_GEOMETRY_SHADER_PRIMITIVES_BIT;
+                }
+                if (EnabledShaderStages & VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT)
+                    QueryPoolCI.pipelineStatistics |= VK_QUERY_PIPELINE_STATISTIC_TESSELLATION_CONTROL_SHADER_PATCHES_BIT;
+                if (EnabledShaderStages & VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT)
+                    QueryPoolCI.pipelineStatistics |= VK_QUERY_PIPELINE_STATISTIC_TESSELLATION_EVALUATION_SHADER_INVOCATIONS_BIT;
+            }
+            break;
+
+            default:
+                UNEXPECTED("Unexpected query type");
+        }
+
+        QueryPoolCI.queryCount = HeapInfo.PoolSize;
+
+        HeapInfo.vkQueryPool = LogicalDevice.CreateQueryPool(QueryPoolCI, "QueryManagerVk: query pool");
+
+        HeapInfo.AvailableQueries.resize(HeapInfo.PoolSize);
+        for (Uint32 i = 0; i < HeapInfo.PoolSize; ++i)
+        {
+            HeapInfo.AvailableQueries[i] = i;
+        }
+    }
+}
+
+QueryManagerVk::~QueryManagerVk()
+{
+    for (Uint32 QueryType = QUERY_TYPE_UNDEFINED + 1; QueryType < QUERY_TYPE_NUM_TYPES; ++QueryType)
+    {
+        auto& HeapInfo = m_Heaps[QueryType];
+        if (HeapInfo.AvailableQueries.size() != HeapInfo.PoolSize)
+        {
+            auto OutstandingQueries = HeapInfo.PoolSize - HeapInfo.AvailableQueries.size();
+            if (OutstandingQueries == 1)
+            {
+                LOG_ERROR_MESSAGE("One query of type ", GetQueryTypeString(static_cast<QUERY_TYPE>(QueryType)),
+                                  " has not been returned to the query manager");
+            }
+            else
+            {
+                LOG_ERROR_MESSAGE(OutstandingQueries, " queries of type ",
+                                  GetQueryTypeString(static_cast<QUERY_TYPE>(QueryType)),
+                                  " have not been returned to the query manager");
+            }
+        }
+    }
+}
+
+Uint32 QueryManagerVk::AllocateQuery(QUERY_TYPE Type)
+{
+    std::lock_guard<std::mutex> Lock(m_HeapMutex);
+
+    Uint32 Index    = InvalidIndex;
+    auto&  HeapInfo = m_Heaps[Type];
+    if (!HeapInfo.AvailableQueries.empty())
+    {
+        Index = HeapInfo.AvailableQueries.front();
+        HeapInfo.AvailableQueries.pop_front();
+    }
+
+    return Index;
+}
+
+void QueryManagerVk::DiscardQuery(QUERY_TYPE Type, Uint32 Index)
+{
+    std::lock_guard<std::mutex> Lock(m_HeapMutex);
+
+    auto& HeapInfo = m_Heaps[Type];
+    VERIFY(Index < HeapInfo.PoolSize, "Query index ", Index, " is out of range");
+#ifdef _DEBUG
+    for (const auto& ind : HeapInfo.AvailableQueries)
+    {
+        VERIFY(ind != Index, "Index ", Index, " already present in available queries list");
+    }
+    for (const auto& ind : HeapInfo.StaleQueries)
+    {
+        VERIFY(ind != Index, "Index ", Index, " already present in stale queries list");
+    }
+#endif
+    HeapInfo.StaleQueries.push_back(Index);
+}
+
+void QueryManagerVk::ResetStaleQueries(VulkanUtilities::VulkanCommandBuffer& CmdBuff)
+{
+    std::lock_guard<std::mutex> Lock(m_HeapMutex);
+
+    for (auto& HeapInfo : m_Heaps)
+    {
+        for (auto& StaleQuery : HeapInfo.StaleQueries)
+        {
+            CmdBuff.ResetQueryPool(HeapInfo.vkQueryPool, StaleQuery, 1);
+            HeapInfo.AvailableQueries.push_back(StaleQuery);
+        }
+        HeapInfo.StaleQueries.clear();
+    }
+}
+
+} // namespace Diligent
diff --git a/Graphics/GraphicsEngineVulkan/src/QueryVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/QueryVkImpl.cpp
index c76b3461..716efa99 100644
--- a/Graphics/GraphicsEngineVulkan/src/QueryVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/QueryVkImpl.cpp
@@ -30,6 +30,7 @@
 #include "QueryVkImpl.h"
 #include "EngineMemory.h"
 #include "RenderDeviceVkImpl.h"
+#include "DeviceContextVkImpl.h"
 
 namespace Diligent
 {
@@ -52,11 +53,168 @@ QueryVkImpl::QueryVkImpl(IReferenceCounters* pRefCounters,
 
 QueryVkImpl::~QueryVkImpl()
 {
+    auto& QueryMgr = m_pContext.RawPtr<DeviceContextVkImpl>()->GetQueryManager();
+    QueryMgr.DiscardQuery(m_Desc.Type, m_QueryPoolIndex);
+}
+
+bool QueryVkImpl::AllocateQuery()
+{
+    auto& QueryMgr = m_pContext.RawPtr<DeviceContextVkImpl>()->GetQueryManager();
+    if (m_QueryPoolIndex != QueryManagerVk::InvalidIndex)
+    {
+        QueryMgr.DiscardQuery(m_Desc.Type, m_QueryPoolIndex);
+    }
+    m_QueryPoolIndex = QueryMgr.AllocateQuery(m_Desc.Type);
+    if (m_QueryPoolIndex == QueryManagerVk::InvalidIndex)
+    {
+        LOG_ERROR_MESSAGE("Failed to allocate Vulkan query for type ", GetQueryTypeString(m_Desc.Type),
+                          ". Increase the query pool size in EngineVkCreateInfo.");
+        return false;
+    }
+
+    return true;
+}
+
+bool QueryVkImpl::OnBeginQuery(IDeviceContext* pContext)
+{
+    if (!TQueryBase::OnBeginQuery(pContext))
+        return false;
+
+    return AllocateQuery();
+}
+
+bool QueryVkImpl::OnEndQuery(IDeviceContext* pContext)
+{
+    if (!TQueryBase::OnEndQuery(pContext))
+        return false;
+
+    if (m_Desc.Type == QUERY_TYPE_TIMESTAMP)
+    {
+        if (!AllocateQuery())
+            return false;
+    }
+
+    if (m_QueryPoolIndex == QueryManagerVk::InvalidIndex)
+    {
+        LOG_ERROR_MESSAGE("Query '", m_Desc.Name, "' is invalid: Vulkan query allocation failed");
+        return false;
+    }
+
+    auto CmdQueueId      = m_pContext.RawPtr<DeviceContextVkImpl>()->GetCommandQueueId();
+    m_QueryEndFenceValue = m_pDevice->GetNextFenceValue(CmdQueueId);
+
+    return true;
 }
 
 bool QueryVkImpl::GetData(void* pData, Uint32 DataSize)
 {
-    return false;
+    auto CmdQueueId          = m_pContext.RawPtr<DeviceContextVkImpl>()->GetCommandQueueId();
+    auto CompletedFenceValue = m_pDevice->GetCompletedFenceValue(CmdQueueId);
+    if (CompletedFenceValue >= m_QueryEndFenceValue)
+    {
+        auto&       QueryMgr      = m_pContext.RawPtr<DeviceContextVkImpl>()->GetQueryManager();
+        const auto& LogicalDevice = m_pDevice->GetLogicalDevice();
+        auto        vkQueryPool   = QueryMgr.GetQueryPool(m_Desc.Type);
+
+        switch (m_Desc.Type)
+        {
+            case QUERY_TYPE_OCCLUSION:
+            {
+                uint64_t Results[2];
+                // If VK_QUERY_RESULT_WITH_AVAILABILITY_BIT is set, the final integer value written for each query
+                // is non-zero if the query's status was available or zero if the status was unavailable.
+
+                // Applications must take care to ensure that use of the VK_QUERY_RESULT_WITH_AVAILABILITY_BIT
+                // bit has the desired effect.
+                // For example, if a query has been used previously and a command buffer records the commands
+                // vkCmdResetQueryPool, vkCmdBeginQuery, and vkCmdEndQuery for that query, then the query will
+                // remain in the available state until vkResetQueryPoolEXT is called or the vkCmdResetQueryPool
+                // command executes on a queue. Applications can use fences or events to ensure that a query has
+                // already been reset before checking for its results or availability status. Otherwise, a stale
+                // value could be returned from a previous use of the query.
+                LogicalDevice.GetQueryPoolResults(vkQueryPool, m_QueryPoolIndex, 1, sizeof(Results), Results, 0, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
+                if (Results[1] == 0)
+                    return false;
+
+                auto& QueryData      = *reinterpret_cast<QueryDataOcclusion*>(pData);
+                QueryData.NumSamples = Results[0];
+            }
+            break;
+
+            case QUERY_TYPE_BINARY_OCCLUSION:
+            {
+                uint64_t Results[2];
+                LogicalDevice.GetQueryPoolResults(vkQueryPool, m_QueryPoolIndex, 1, sizeof(Results), Results, 0, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
+                if (Results[1] == 0)
+                    return false;
+
+                auto& QueryData           = *reinterpret_cast<QueryDataBinaryOcclusion*>(pData);
+                QueryData.AnySamplePassed = Results[0] != 0;
+            }
+            break;
+
+            case QUERY_TYPE_TIMESTAMP:
+            {
+                uint64_t Results[2];
+                LogicalDevice.GetQueryPoolResults(vkQueryPool, m_QueryPoolIndex, 1, sizeof(Results), Results, 0, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
+                if (Results[1] == 0)
+                    return false;
+
+                auto& QueryData     = *reinterpret_cast<QueryDataTimestamp*>(pData);
+                QueryData.Counter   = Results[0];
+                QueryData.Frequency = QueryMgr.GetCounterFrequency();
+            }
+            break;
+
+            case QUERY_TYPE_PIPELINE_STATISTICS:
+            {
+                // Pipeline statistics queries write one integer value for each bit that is enabled in the
+                // pipelineStatistics when the pool is created, and the statistics values are written in bit
+                // order starting from the least significant bit. (17.2)
+
+                Uint64 Results[12];
+                LogicalDevice.GetQueryPoolResults(vkQueryPool, m_QueryPoolIndex, 1, sizeof(Results), Results, 0, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT);
+                auto& QueryData = *reinterpret_cast<QueryDataPipelineStatistics*>(pData);
+
+                const auto EnabledShaderStages = LogicalDevice.GetEnabledGraphicsShaderStages();
+
+                auto Idx = 0;
+
+                QueryData.InputVertices   = Results[Idx++]; // INPUT_ASSEMBLY_VERTICES_BIT   = 0x00000001
+                QueryData.InputPrimitives = Results[Idx++]; // INPUT_ASSEMBLY_PRIMITIVES_BIT = 0x00000002
+                QueryData.VSInvocations   = Results[Idx++]; // VERTEX_SHADER_INVOCATIONS_BIT = 0x00000004
+                if (EnabledShaderStages & VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT)
+                {
+                    QueryData.GSInvocations = Results[Idx++]; // GEOMETRY_SHADER_INVOCATIONS_BIT = 0x00000008
+                    QueryData.GSPrimitives  = Results[Idx++]; // GEOMETRY_SHADER_PRIMITIVES_BIT  = 0x00000010
+                }
+                QueryData.ClippingInvocations = Results[Idx++]; // CLIPPING_INVOCATIONS_BIT         = 0x00000020
+                QueryData.ClippingPrimitives  = Results[Idx++]; // CLIPPING_PRIMITIVES_BIT          = 0x00000040
+                QueryData.PSInvocations       = Results[Idx++]; // FRAGMENT_SHADER_INVOCATIONS_BIT  = 0x00000080
+
+                if (EnabledShaderStages & VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT)
+                    QueryData.HSInvocations = Results[Idx++]; // TESSELLATION_CONTROL_SHADER_PATCHES_BIT        = 0x00000100
+
+                if (EnabledShaderStages & VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT)
+                    QueryData.DSInvocations = Results[Idx++]; // TESSELLATION_EVALUATION_SHADER_INVOCATIONS_BIT = 0x00000200
+
+                QueryData.CSInvocations = Results[Idx++]; // COMPUTE_SHADER_INVOCATIONS_BIT = 0x00000400
+
+                if (!Results[Idx])
+                    return false;
+            }
+            break;
+
+            default:
+                UNEXPECTED("Unexpected query type");
+        }
+
+        return true;
+    }
+    else
+    {
+        return false;
+    }
 }
 
 } // namespace Diligent