D3D12 backend: few updates to mesh shader implementation

8 files changed, 88 insertions, 48 deletions

diff --git a/Graphics/GraphicsEngineD3D12/CMakeLists.txt b/Graphics/GraphicsEngineD3D12/CMakeLists.txt
index e8a48236..28e7abad 100644
--- a/Graphics/GraphicsEngineD3D12/CMakeLists.txt
+++ b/Graphics/GraphicsEngineD3D12/CMakeLists.txt
@@ -181,10 +181,10 @@ target_compile_definitions(Diligent-GraphicsEngineD3D12-shared PUBLIC ENGINE_DLL
 
 if(${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION} STRGREATER_EQUAL "10.0.19041.0")
     set(D3D12_H_HAS_MESH_SHADER ON CACHE INTERNAL "" FORCE)
-    target_compile_definitions(Diligent-GraphicsEngineD3D12-static PRIVATE D3D12_H_HAS_MESH_SHADER)
+    target_compile_definitions(Diligent-GraphicsEngineD3D12-static PRIVATE D3D12_H_HAS_MESH_SHADER=1)
 endif()
 if(${DILIGENT_HAS_D3D12_DXIL_COMPILER})
-    target_compile_definitions(Diligent-GraphicsEngineD3D12-static PRIVATE DILIGENT_HAS_D3D12_DXIL_COMPILER)
+    target_compile_definitions(Diligent-GraphicsEngineD3D12-static PRIVATE DILIGENT_HAS_D3D12_DXIL_COMPILE=1)
 endif()
 
 # Set output name to GraphicsEngineD3D12_{32|64}{r|d}
diff --git a/Graphics/GraphicsEngineD3D12/include/RenderDeviceD3D12Impl.hpp b/Graphics/GraphicsEngineD3D12/include/RenderDeviceD3D12Impl.hpp
index bac52b9e..710db2ef 100644
--- a/Graphics/GraphicsEngineD3D12/include/RenderDeviceD3D12Impl.hpp
+++ b/Graphics/GraphicsEngineD3D12/include/RenderDeviceD3D12Impl.hpp
@@ -192,7 +192,7 @@ private:
 
     QueryManagerD3D12 m_QueryMgr;
 
-    D3D_SHADER_MODEL m_ShaderModel = D3D_SHADER_MODEL_5_1;
+    D3D_SHADER_MODEL m_MaxShaderModel = D3D_SHADER_MODEL_5_1;
 
     std::unique_ptr<IDxCompilerLibrary> m_pDxCompiler;
 };
diff --git a/Graphics/GraphicsEngineD3D12/include/ShaderResourcesD3D12.hpp b/Graphics/GraphicsEngineD3D12/include/ShaderResourcesD3D12.hpp
index b1c12f90..228db950 100644
--- a/Graphics/GraphicsEngineD3D12/include/ShaderResourcesD3D12.hpp
+++ b/Graphics/GraphicsEngineD3D12/include/ShaderResourcesD3D12.hpp
@@ -92,10 +92,10 @@ class ShaderResourcesD3D12 final : public ShaderResources
 {
 public:
     // Loads shader resources from the compiled shader bytecode
-    ShaderResourcesD3D12(ID3DBlob*                    pShaderBytecode,
-                         const ShaderDesc&            ShdrDesc,
-                         const char*                  CombinedSamplerSuffix,
-                         class RenderDeviceD3D12Impl* pRenderDeviceD3D12);
+    ShaderResourcesD3D12(ID3DBlob*                 pShaderBytecode,
+                         const ShaderDesc&         ShdrDesc,
+                         const char*               CombinedSamplerSuffix,
+                         class IDxCompilerLibrary* pCompilerLibrary);
 
     // clang-format off
     ShaderResourcesD3D12             (const ShaderResourcesD3D12&)  = delete;
diff --git a/Graphics/GraphicsEngineD3D12/src/DeviceContextD3D12Impl.cpp b/Graphics/GraphicsEngineD3D12/src/DeviceContextD3D12Impl.cpp
index 70289385..546c9de8 100644
--- a/Graphics/GraphicsEngineD3D12/src/DeviceContextD3D12Impl.cpp
+++ b/Graphics/GraphicsEngineD3D12/src/DeviceContextD3D12Impl.cpp
@@ -227,6 +227,8 @@ void DeviceContextD3D12Impl::SetPipelineState(IPipelineState* pPipelineState)
     }
     else
     {
+        VERIFY_EXPR(PSODesc.IsAnyGraphicsPipeline());
+
         auto& GraphicsCtx = CmdCtx.AsGraphicsContext();
         GraphicsCtx.SetPipelineState(pd3d12PSO);
 
diff --git a/Graphics/GraphicsEngineD3D12/src/RenderDeviceD3D12Impl.cpp b/Graphics/GraphicsEngineD3D12/src/RenderDeviceD3D12Impl.cpp
index bdf596a4..76e8aa07 100644
--- a/Graphics/GraphicsEngineD3D12/src/RenderDeviceD3D12Impl.cpp
+++ b/Graphics/GraphicsEngineD3D12/src/RenderDeviceD3D12Impl.cpp
@@ -67,7 +67,7 @@ static CComPtr<IDXGIAdapter1> DXGIAdapterFromD3D12Device(ID3D12Device* pd3d12Dev
 
 ShaderVersion RenderDeviceD3D12Impl::GetMaxShaderModel() const
 {
-    return ShaderVersion{Uint8((m_ShaderModel >> 4) & 0xF), Uint8(m_ShaderModel & 0xF)};
+    return ShaderVersion{static_cast<Uint8>((m_MaxShaderModel >> 4) & 0xF), static_cast<Uint8>(m_MaxShaderModel & 0xF)};
 }
 
 D3D_FEATURE_LEVEL RenderDeviceD3D12Impl::GetD3DFeatureLevel() const
@@ -206,24 +206,25 @@ RenderDeviceD3D12Impl::RenderDeviceD3D12Impl(IReferenceCounters*          pRefCo
     {
         // Direct3D12 supports shader model 5.1 on all feature levels.
         // https://docs.microsoft.com/en-us/windows/win32/direct3d12/hardware-feature-levels#feature-level-support
-        m_ShaderModel = D3D_SHADER_MODEL_5_1;
+        m_MaxShaderModel = D3D_SHADER_MODEL_5_1;
 
         // Header may not have constants for D3D_SHADER_MODEL_6_1 and above.
-        const D3D_SHADER_MODEL Models[] = {
-            D3D_SHADER_MODEL(0x65), // for mesh shader
-            D3D_SHADER_MODEL(0x64),
-            D3D_SHADER_MODEL(0x63),
-            D3D_SHADER_MODEL(0x62),
-            D3D_SHADER_MODEL(0x61),
-            D3D_SHADER_MODEL_6_0};
-
-        D3D12_FEATURE_DATA_SHADER_MODEL ShaderModel = {};
+        const D3D_SHADER_MODEL Models[] = //
+            {
+                static_cast<D3D_SHADER_MODEL>(0x65), // minimum required for mesh shader
+                static_cast<D3D_SHADER_MODEL>(0x64),
+                static_cast<D3D_SHADER_MODEL>(0x63),
+                static_cast<D3D_SHADER_MODEL>(0x62),
+                static_cast<D3D_SHADER_MODEL>(0x61),
+                D3D_SHADER_MODEL_6_0 //
+            };
+
         for (auto Model : Models)
         {
-            ShaderModel.HighestShaderModel = Model;
+            D3D12_FEATURE_DATA_SHADER_MODEL ShaderModel = {Model};
             if (SUCCEEDED(m_pd3d12Device->CheckFeatureSupport(D3D12_FEATURE_SHADER_MODEL, &ShaderModel, sizeof(ShaderModel))))
             {
-                m_ShaderModel = ShaderModel.HighestShaderModel;
+                m_MaxShaderModel = ShaderModel.HighestShaderModel;
                 break;
             }
         }
@@ -232,11 +233,13 @@ RenderDeviceD3D12Impl::RenderDeviceD3D12Impl(IReferenceCounters*          pRefCo
     // Check if mesh shader is supported.
 #ifdef D3D12_H_HAS_MESH_SHADER
     {
-        D3D12_FEATURE_DATA_D3D12_OPTIONS7 FeatureData        = {};
-        bool                              SupportsMeshShader = SUCCEEDED(m_pd3d12Device->CheckFeatureSupport(D3D12_FEATURE_D3D12_OPTIONS7, &FeatureData, sizeof(FeatureData))) &&
+        D3D12_FEATURE_DATA_D3D12_OPTIONS7 FeatureData = {};
+
+        bool SupportsMeshShader =
+            SUCCEEDED(m_pd3d12Device->CheckFeatureSupport(D3D12_FEATURE_D3D12_OPTIONS7, &FeatureData, sizeof(FeatureData))) &&
             FeatureData.MeshShaderTier != D3D12_MESH_SHADER_TIER_NOT_SUPPORTED;
 
-        m_DeviceCaps.Features.MeshShaders = (m_ShaderModel >= D3D_SHADER_MODEL_6_5 && SupportsMeshShader);
+        m_DeviceCaps.Features.MeshShaders = (m_MaxShaderModel >= D3D_SHADER_MODEL_6_5 && SupportsMeshShader);
     }
 #endif
 
diff --git a/Graphics/GraphicsEngineD3D12/src/RootSignature.cpp b/Graphics/GraphicsEngineD3D12/src/RootSignature.cpp
index 0acaa810..6ec0497f 100644
--- a/Graphics/GraphicsEngineD3D12/src/RootSignature.cpp
+++ b/Graphics/GraphicsEngineD3D12/src/RootSignature.cpp
@@ -175,7 +175,7 @@ RootSignature::RootSignature() :
 }
 
 // clang-format off
-static D3D12_SHADER_VISIBILITY ShaderTypeInd2ShaderVisibilityMap[]
+static constexpr D3D12_SHADER_VISIBILITY ShaderTypeInd2ShaderVisibilityMap[]
 {
     D3D12_SHADER_VISIBILITY_VERTEX,        // 0
     D3D12_SHADER_VISIBILITY_PIXEL,         // 1
diff --git a/Graphics/GraphicsEngineD3D12/src/ShaderD3D12Impl.cpp b/Graphics/GraphicsEngineD3D12/src/ShaderD3D12Impl.cpp
index 80025b36..36a14167 100644
--- a/Graphics/GraphicsEngineD3D12/src/ShaderD3D12Impl.cpp
+++ b/Graphics/GraphicsEngineD3D12/src/ShaderD3D12Impl.cpp
@@ -38,23 +38,50 @@ namespace Diligent
 
 static ShaderVersion GetD3D12ShaderModel(RenderDeviceD3D12Impl* pDevice, const ShaderVersion& HLSLVersion, SHADER_COMPILER ShaderCompiler)
 {
-    if (ShaderCompiler != SHADER_COMPILER_DXC)
-        return HLSLVersion.Major == 0 ? ShaderVersion{5, 1} : HLSLVersion;
+    ShaderVersion CompilerSM;
+    if (ShaderCompiler == SHADER_COMPILER_DXC)
+    {
+        if (pDevice->GetDxCompiler() && pDevice->GetDxCompiler()->IsLoaded())
+        {
+            CompilerSM = pDevice->GetDxCompiler()->GetMaxShaderModel();
+        }
+        else
+        {
+            LOG_ERROR_MESSAGE("DXC compiler is not loaded");
+            CompilerSM = ShaderVersion{5, 1};
+        }
+    }
+    else
+    {
+        VERIFY(ShaderCompiler == SHADER_COMPILER_FXC || ShaderCompiler == SHADER_COMPILER_DEFAULT, "Unexpected compiler");
+        // Direct3D12 supports shader model 5.1 on all feature levels.
+        // https://docs.microsoft.com/en-us/windows/win32/direct3d12/hardware-feature-levels#feature-level-support
+        CompilerSM = ShaderVersion{5, 1};
+    }
 
-    ShaderVersion DeviceSM   = pDevice->GetMaxShaderModel();
-    ShaderVersion CompilerSM = pDevice->GetDxCompiler() && pDevice->GetDxCompiler()->IsLoaded() ? pDevice->GetDxCompiler()->GetMaxShaderModel() : ShaderVersion{5, 1};
-    ShaderVersion MaxSM;
+    ShaderVersion DeviceSM = pDevice->GetMaxShaderModel();
 
-    MaxSM = DeviceSM.Major == CompilerSM.Major ?
-        (DeviceSM.Minor > CompilerSM.Minor ? CompilerSM : DeviceSM) :
-        (DeviceSM.Major > CompilerSM.Major ? CompilerSM : DeviceSM);
+    ShaderVersion MaxSupportedSM = DeviceSM.Major == CompilerSM.Major ?
+        (DeviceSM.Minor < CompilerSM.Minor ? DeviceSM : CompilerSM) :
+        (DeviceSM.Major < CompilerSM.Major ? DeviceSM : CompilerSM);
 
     if (HLSLVersion.Major == 0 && HLSLVersion.Minor == 0)
-        return MaxSM;
-
-    return HLSLVersion.Major == MaxSM.Major ?
-        (HLSLVersion.Minor > MaxSM.Minor ? MaxSM : HLSLVersion) :
-        (HLSLVersion.Major > MaxSM.Major ? MaxSM : HLSLVersion);
+    {
+        return MaxSupportedSM;
+    }
+    else
+    {
+        if (HLSLVersion.Major > MaxSupportedSM.Major ||
+            HLSLVersion.Major == MaxSupportedSM.Major && HLSLVersion.Minor > MaxSupportedSM.Minor)
+        {
+            LOG_WARNING_MESSAGE("Requested shader model ", Uint32{HLSLVersion.Major}, '_', Uint32{HLSLVersion.Minor},
+                                " is not supported by the device/compiler. Downgrading to maximum supported version ",
+                                Uint32{MaxSupportedSM.Major}, '_', Uint32{MaxSupportedSM.Minor});
+            return MaxSupportedSM;
+        }
+        else
+            return HLSLVersion;
+    }
 }
 
 ShaderD3D12Impl::ShaderD3D12Impl(IReferenceCounters*     pRefCounters,
@@ -73,7 +100,13 @@ ShaderD3D12Impl::ShaderD3D12Impl(IReferenceCounters*     pRefCounters,
     // Load shader resources
     auto& Allocator  = GetRawAllocator();
     auto* pRawMem    = ALLOCATE(Allocator, "Allocator for ShaderResources", ShaderResourcesD3D12, 1);
-    auto* pResources = new (pRawMem) ShaderResourcesD3D12(m_pShaderByteCode, m_Desc, ShaderCI.UseCombinedTextureSamplers ? ShaderCI.CombinedSamplerSuffix : nullptr, pRenderDeviceD3D12);
+    auto* pResources = new (pRawMem) ShaderResourcesD3D12 //
+        {
+            m_pShaderByteCode,
+            m_Desc,
+            ShaderCI.UseCombinedTextureSamplers ? ShaderCI.CombinedSamplerSuffix : nullptr,
+            pRenderDeviceD3D12->GetDxCompiler() //
+        };
     m_pShaderResources.reset(pResources, STDDeleterRawMem<ShaderResourcesD3D12>(Allocator));
 }
 
diff --git a/Graphics/GraphicsEngineD3D12/src/ShaderResourcesD3D12.cpp b/Graphics/GraphicsEngineD3D12/src/ShaderResourcesD3D12.cpp
index 01726d00..6cd31660 100644
--- a/Graphics/GraphicsEngineD3D12/src/ShaderResourcesD3D12.cpp
+++ b/Graphics/GraphicsEngineD3D12/src/ShaderResourcesD3D12.cpp
@@ -32,16 +32,15 @@
 #include "ShaderD3DBase.hpp"
 #include "ShaderBase.hpp"
 #include "DXILUtils.hpp"
-#include "RenderDeviceD3D12Impl.hpp"
 
 namespace Diligent
 {
 
 
-ShaderResourcesD3D12::ShaderResourcesD3D12(ID3DBlob*              pShaderBytecode,
-                                           const ShaderDesc&      ShdrDesc,
-                                           const char*            CombinedSamplerSuffix,
-                                           RenderDeviceD3D12Impl* pRenderDeviceD3D12) :
+ShaderResourcesD3D12::ShaderResourcesD3D12(ID3DBlob*                 pShaderBytecode,
+                                           const ShaderDesc&         ShdrDesc,
+                                           const char*               CombinedSamplerSuffix,
+                                           class IDxCompilerLibrary* pCompilerLibrary) :
     ShaderResources{ShdrDesc.ShaderType}
 {
     class NewResourceHandler
@@ -58,18 +57,21 @@ ShaderResourcesD3D12::ShaderResourcesD3D12(ID3DBlob*              pShaderBytecod
     };
 
     CComPtr<ID3D12ShaderReflection> pShaderReflection;
-    auto*                           DxCompiler = pRenderDeviceD3D12->GetDxCompiler();
+    if (pCompilerLibrary != nullptr)
+    {
+        // Try to get shader reflection with DXC.
+        DxcGetShaderReflection(pCompilerLibrary, reinterpret_cast<IDxcBlob*>(pShaderBytecode), &pShaderReflection);
+    }
 
-    // At first try to get shader reflection with a DXC.
-    if (!DxcGetShaderReflection(DxCompiler, reinterpret_cast<IDxcBlob*>(pShaderBytecode), &pShaderReflection))
+    if (!pShaderReflection)
     {
         // Use FXC to get reflection.
-        HRESULT hr = D3DReflect(pShaderBytecode->GetBufferPointer(), pShaderBytecode->GetBufferSize(), __uuidof(pShaderReflection), reinterpret_cast<void**>(&pShaderReflection));
+        auto hr = D3DReflect(pShaderBytecode->GetBufferPointer(), pShaderBytecode->GetBufferSize(), __uuidof(pShaderReflection), reinterpret_cast<void**>(&pShaderReflection));
         CHECK_D3D_RESULT_THROW(hr, "Failed to get the shader reflection");
     }
 
     Initialize<D3D12_SHADER_DESC, D3D12_SHADER_INPUT_BIND_DESC, ID3D12ShaderReflection>(
-        static_cast<ID3D12ShaderReflection*>(pShaderReflection),
+        pShaderReflection,
         NewResourceHandler{},
         ShdrDesc.Name,
         CombinedSamplerSuffix);