Refactoring shader compilation - part III

7 files changed, 126 insertions, 163 deletions

diff --git a/Graphics/GraphicsEngineD3DBase/src/ShaderD3DBase.cpp b/Graphics/GraphicsEngineD3DBase/src/ShaderD3DBase.cpp
index 97691879..39684476 100644
--- a/Graphics/GraphicsEngineD3DBase/src/ShaderD3DBase.cpp
+++ b/Graphics/GraphicsEngineD3DBase/src/ShaderD3DBase.cpp
@@ -44,62 +44,6 @@
 namespace Diligent
 {
 
-static HRESULT CompileDxilShader(IDXCompiler*            DxCompiler,
-                                 const char*             Source,
-                                 size_t                  SourceLength,
-                                 const ShaderCreateInfo& ShaderCI,
-                                 LPCSTR                  profile,
-                                 ID3DBlob**              ppBlobOut,
-                                 ID3DBlob**              ppCompilerOutput)
-{
-    VERIFY_EXPR(DxCompiler != nullptr);
-
-    std::vector<std::unique_ptr<wchar_t[]>> StringPool;
-
-    auto UTF8ToUTF16 = [&StringPool](LPCSTR lpUTF8) //
-    {
-        // When last parameter is 0, the function returns the required buffer size, in characters,
-        // including any terminating null character.
-        const auto                 nChars = MultiByteToWideChar(CP_UTF8, 0, lpUTF8, -1, NULL, 0);
-        std::unique_ptr<wchar_t[]> wstr{new wchar_t[nChars]};
-        MultiByteToWideChar(CP_UTF8, 0, lpUTF8, -1, wstr.get(), nChars);
-        StringPool.emplace_back(std::move(wstr));
-        return StringPool.back().get();
-    };
-
-    const wchar_t* pArgs[] =
-        {
-            L"-Zpc", // Matrices in column-major order
-                     //L"-WX",  // Warnings as errors
-#ifdef DILIGENT_DEBUG
-            L"-Zi", // Debug info
-            //L"-Qembed_debug", // Embed debug info into the shader (some compilers do not recognize this flag)
-            L"-Od", // Disable optimization
-#else
-            L"-Od", // TODO: something goes wrong if optimization is enabled
-                    //L"-O3", // Optimization level 3
-#endif
-        };
-
-    VERIFY_EXPR(__uuidof(ID3DBlob) == __uuidof(IDxcBlob));
-
-    IDXCompiler::CompileAttribs CA;
-    CA.Source                     = Source;
-    CA.SourceLength               = static_cast<Uint32>(SourceLength);
-    CA.EntryPoint                 = UTF8ToUTF16(ShaderCI.EntryPoint);
-    CA.Profile                    = UTF8ToUTF16(profile);
-    CA.pArgs                      = pArgs;
-    CA.ArgsCount                  = _countof(pArgs);
-    CA.pShaderSourceStreamFactory = ShaderCI.pShaderSourceStreamFactory;
-    CA.ppBlobOut                  = reinterpret_cast<IDxcBlob**>(ppBlobOut);
-    CA.ppCompilerOutput           = reinterpret_cast<IDxcBlob**>(ppCompilerOutput);
-    if (!DxCompiler->Compile(CA))
-    {
-        return E_FAIL;
-    }
-    return S_OK;
-}
-
 class D3DIncludeImpl : public ID3DInclude
 {
 public:
@@ -156,12 +100,12 @@ static HRESULT CompileShader(const char*             Source,
     // the release configuration of this program.
     dwShaderFlags |= D3DCOMPILE_DEBUG;
 #else
-                    // Warning: do not use this flag as it causes shader compiler to fail the compilation and
-                    // report strange errors:
-                    // dwShaderFlags |= D3D10_SHADER_OPTIMIZATION_LEVEL3;
+    // Warning: do not use this flag as it causes shader compiler to fail the compilation and
+    // report strange errors:
+    // dwShaderFlags |= D3D10_SHADER_OPTIMIZATION_LEVEL3;
 #endif
 
-    D3DIncludeImpl IncludeImpl(ShaderCI.pShaderSourceStreamFactory);
+    D3DIncludeImpl IncludeImpl{ShaderCI.pShaderSourceStreamFactory};
     return D3DCompile(Source, SourceLength, NULL, nullptr, &IncludeImpl, ShaderCI.EntryPoint, profile, dwShaderFlags, 0, ppBlobOut, ppCompilerOutput);
 }
 
@@ -181,66 +125,51 @@ ShaderD3DBase::ShaderD3DBase(const ShaderCreateInfo& ShaderCI, const ShaderVersi
             case SHADER_COMPILER_DEFAULT: UseDXC = false;                                            break;
             case SHADER_COMPILER_DXC:     UseDXC = DxCompiler != nullptr && DxCompiler->IsLoaded();  break;
             case SHADER_COMPILER_FXC:     UseDXC = false;                                            break;
-                // clang-format on
+            // clang-format on
             default: UNEXPECTED("Unsupported shader compiler");
         }
 
-        std::string strShaderProfile;
-        switch (ShaderCI.Desc.ShaderType)
+        if (UseDXC)
         {
-            // clang-format off
-            case SHADER_TYPE_VERTEX:        strShaderProfile="vs"; break;
-            case SHADER_TYPE_PIXEL:         strShaderProfile="ps"; break;
-            case SHADER_TYPE_GEOMETRY:      strShaderProfile="gs"; break;
-            case SHADER_TYPE_HULL:          strShaderProfile="hs"; break;
-            case SHADER_TYPE_DOMAIN:        strShaderProfile="ds"; break;
-            case SHADER_TYPE_COMPUTE:       strShaderProfile="cs"; break;
-            case SHADER_TYPE_AMPLIFICATION: strShaderProfile="as"; break;
-            case SHADER_TYPE_MESH:          strShaderProfile="ms"; break;
-                // clang-format on
-            default: UNEXPECTED("Unknown shader type");
+            VERIFY_EXPR(__uuidof(ID3DBlob) == __uuidof(IDxcBlob));
+            DxCompiler->Compile(ShaderCI, nullptr, reinterpret_cast<IDxcBlob**>(&m_pShaderByteCode), nullptr, ShaderCI.ppCompilerOutput);
         }
+        else
+        {
+            std::string strShaderProfile = GetHLSLProfileString(ShaderCI.Desc.ShaderType, ShaderModel);
 
-        strShaderProfile += "_";
-        strShaderProfile += std::to_string(ShaderModel.Major);
-        strShaderProfile += "_";
-        strShaderProfile += std::to_string(ShaderModel.Minor);
+            String ShaderSource = BuildHLSLSourceString(ShaderCI);
 
-        String ShaderSource = BuildHLSLSourceString(ShaderCI);
+            DEV_CHECK_ERR(ShaderCI.EntryPoint != nullptr, "Entry point must not be null");
 
-        DEV_CHECK_ERR(ShaderCI.EntryPoint != nullptr, "Entry point must not be null");
+            CComPtr<ID3DBlob> errors;
 
-        CComPtr<ID3DBlob> errors;
-        HRESULT           hr;
+            auto hr = CompileShader(ShaderSource.c_str(), ShaderSource.length(), ShaderCI, strShaderProfile.c_str(), &m_pShaderByteCode, &errors);
 
-        if (UseDXC)
-            hr = CompileDxilShader(DxCompiler, ShaderSource.c_str(), ShaderSource.length(), ShaderCI, strShaderProfile.c_str(), &m_pShaderByteCode, &errors);
-        else
-            hr = CompileShader(ShaderSource.c_str(), ShaderSource.length(), ShaderCI, strShaderProfile.c_str(), &m_pShaderByteCode, &errors);
+            const size_t CompilerMsgLen = errors ? errors->GetBufferSize() : 0;
+            const char*  CompilerMsg    = CompilerMsgLen > 0 ? static_cast<const char*>(errors->GetBufferPointer()) : nullptr;
 
-        const size_t CompilerMsgLen = errors ? errors->GetBufferSize() : 0;
-        const char*  CompilerMsg    = CompilerMsgLen > 0 ? static_cast<const char*>(errors->GetBufferPointer()) : nullptr;
-
-        if (CompilerMsg != nullptr && ShaderCI.ppCompilerOutput != nullptr)
-        {
-            auto* pOutputDataBlob = MakeNewRCObj<DataBlobImpl>()(CompilerMsgLen + 1 + ShaderSource.length() + 1);
-            char* DataPtr         = static_cast<char*>(pOutputDataBlob->GetDataPtr());
-            memcpy(DataPtr, CompilerMsg, CompilerMsgLen);
-            DataPtr[CompilerMsgLen] = 0; // Set null terminator
-            memcpy(DataPtr + CompilerMsgLen + 1, ShaderSource.data(), ShaderSource.length() + 1);
-            pOutputDataBlob->QueryInterface(IID_DataBlob, reinterpret_cast<IObject**>(ShaderCI.ppCompilerOutput));
-        }
-
-        if (FAILED(hr))
-        {
-            ComErrorDesc ErrDesc(hr);
-            if (ShaderCI.ppCompilerOutput != nullptr)
+            if (CompilerMsg != nullptr && ShaderCI.ppCompilerOutput != nullptr)
             {
-                LOG_ERROR_AND_THROW("Failed to compile D3D shader \"", (ShaderCI.Desc.Name != nullptr ? ShaderCI.Desc.Name : ""), "\" (", ErrDesc.Get(), ").");
+                auto* pOutputDataBlob = MakeNewRCObj<DataBlobImpl>()(CompilerMsgLen + 1 + ShaderSource.length() + 1);
+                char* DataPtr         = static_cast<char*>(pOutputDataBlob->GetDataPtr());
+                memcpy(DataPtr, CompilerMsg, CompilerMsgLen);
+                DataPtr[CompilerMsgLen] = 0; // Set null terminator
+                memcpy(DataPtr + CompilerMsgLen + 1, ShaderSource.data(), ShaderSource.length() + 1);
+                pOutputDataBlob->QueryInterface(IID_DataBlob, reinterpret_cast<IObject**>(ShaderCI.ppCompilerOutput));
             }
-            else
+
+            if (FAILED(hr))
             {
-                LOG_ERROR_AND_THROW("Failed to compile D3D shader \"", (ShaderCI.Desc.Name != nullptr ? ShaderCI.Desc.Name : ""), "\" (", ErrDesc.Get(), "):\n", (CompilerMsg != nullptr ? CompilerMsg : "<no compiler log available>"));
+                ComErrorDesc ErrDesc(hr);
+                if (ShaderCI.ppCompilerOutput != nullptr)
+                {
+                    LOG_ERROR_AND_THROW("Failed to compile D3D shader \"", (ShaderCI.Desc.Name != nullptr ? ShaderCI.Desc.Name : ""), "\" (", ErrDesc.Get(), ").");
+                }
+                else
+                {
+                    LOG_ERROR_AND_THROW("Failed to compile D3D shader \"", (ShaderCI.Desc.Name != nullptr ? ShaderCI.Desc.Name : ""), "\" (", ErrDesc.Get(), "):\n", (CompilerMsg != nullptr ? CompilerMsg : "<no compiler log available>"));
+                }
             }
         }
     }
diff --git a/Graphics/GraphicsEngineVulkan/src/ShaderVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/ShaderVkImpl.cpp
index 733f5b74..033f711a 100644
--- a/Graphics/GraphicsEngineVulkan/src/ShaderVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/ShaderVkImpl.cpp
@@ -67,8 +67,18 @@ ShaderVkImpl::ShaderVkImpl(IReferenceCounters*     pRefCounters,
         switch (CreationAttribs.ShaderCompiler)
         {
             case SHADER_COMPILER_DXC:
-                m_SPIRV = DXILtoSPIRV(pRenderDeviceVk->GetDxCompiler(), CreationAttribs, VulkanDefine, CreationAttribs.ppCompilerOutput);
-                break;
+            {
+                auto* pDXComiler = pRenderDeviceVk->GetDxCompiler();
+                if (pDXComiler != nullptr && pDXComiler->IsLoaded())
+                {
+                    pDXComiler->Compile(CreationAttribs, VulkanDefine, nullptr, &m_SPIRV, CreationAttribs.ppCompilerOutput);
+                }
+                else
+                {
+                    LOG_ERROR_AND_THROW("DX Compiler is not loaded");
+                }
+            }
+            break;
 
             case SHADER_COMPILER_DEFAULT:
             case SHADER_COMPILER_GLSLANG:
diff --git a/Graphics/ShaderTools/include/DXCompiler.hpp b/Graphics/ShaderTools/include/DXCompiler.hpp
index 10d60be7..d361361c 100644
--- a/Graphics/ShaderTools/include/DXCompiler.hpp
+++ b/Graphics/ShaderTools/include/DXCompiler.hpp
@@ -73,6 +73,12 @@ public:
     };
     virtual bool Compile(const CompileAttribs& Attribs) = 0;
 
+    virtual void Compile(const ShaderCreateInfo& ShaderCI,
+                         const char*             ExtraDefinitions,
+                         IDxcBlob**              ppByteCodeBlob,
+                         std::vector<uint32_t>*  pByteCode,
+                         IDataBlob**             ppCompilerOutput) noexcept(false) = 0;
+
     // Attempts to extract shader reflection from the bytecode using DXC.
     virtual void GetD3D12ShaderReflection(IDxcBlob*                pShaderBytecode,
                                           ID3D12ShaderReflection** ppShaderReflection) = 0;
@@ -83,12 +89,4 @@ public:
 IDXCompiler* CreateDXCompiler(DXCompilerTarget Target, const char* pLibraryName);
 
 
-#if VULKAN_SUPPORTED
-std::vector<uint32_t> DXILtoSPIRV(IDXCompiler*            pLibrary,
-                                  const ShaderCreateInfo& Attribs,
-                                  const char*             ExtraDefinitions,
-                                  IDataBlob**             ppCompilerOutput) noexcept(false);
-#endif
-
-
 } // namespace Diligent
diff --git a/Graphics/ShaderTools/include/HLSLUtils.hpp b/Graphics/ShaderTools/include/HLSLUtils.hpp
index 8d8774f5..9abd1e7d 100644
--- a/Graphics/ShaderTools/include/HLSLUtils.hpp
+++ b/Graphics/ShaderTools/include/HLSLUtils.hpp
@@ -37,4 +37,6 @@ namespace Diligent
 String BuildHLSLSourceString(const ShaderCreateInfo& ShaderCI,
                              const char*             ExtraDefinitions = nullptr);
 
+String GetHLSLProfileString(SHADER_TYPE ShaderType, ShaderVersion ShaderModel);
+
 } // namespace Diligent
diff --git a/Graphics/ShaderTools/src/DXCompiler.cpp b/Graphics/ShaderTools/src/DXCompiler.cpp
index 80235f65..3ce6066f 100644
--- a/Graphics/ShaderTools/src/DXCompiler.cpp
+++ b/Graphics/ShaderTools/src/DXCompiler.cpp
@@ -83,6 +83,12 @@ public:
 
     bool Compile(const CompileAttribs& Attribs) override final;
 
+    virtual void Compile(const ShaderCreateInfo& ShaderCI,
+                         const char*             ExtraDefinitions,
+                         IDxcBlob**              ppByteCodeBlob,
+                         std::vector<uint32_t>*  pByteCode,
+                         IDataBlob**             ppCompilerOutput) noexcept(false) override final;
+
     virtual void GetD3D12ShaderReflection(IDxcBlob*                pShaderBytecode,
                                           ID3D12ShaderReflection** ppShaderReflection) override final;
 
@@ -404,18 +410,21 @@ void DXCompilerImpl::GetD3D12ShaderReflection(IDxcBlob*                pShaderBy
 
 
 
-#if VULKAN_SUPPORTED
-
-std::vector<uint32_t> DXILtoSPIRV(IDXCompiler*            pLibrary,
-                                  const ShaderCreateInfo& ShaderCI,
-                                  const char*             ExtraDefinitions,
-                                  IDataBlob**             ppCompilerOutput) noexcept(false)
+void DXCompilerImpl::Compile(const ShaderCreateInfo& ShaderCI,
+                             const char*             ExtraDefinitions,
+                             IDxcBlob**              ppByteCodeBlob,
+                             std::vector<uint32_t>*  pByteCode,
+                             IDataBlob**             ppCompilerOutput) noexcept(false)
 {
-
+    if (!IsLoaded())
+    {
+        UNEXPECTED("DX compiler is not loaded");
+        return;
+    }
 
     // validate shader version
     ShaderVersion ShaderModel = ShaderCI.HLSLVersion;
-    ShaderVersion MaxSM       = pLibrary->GetMaxShaderModel();
+    ShaderVersion MaxSM       = GetMaxShaderModel();
 
     if (ShaderModel.Major < 6 || ShaderModel.Major > MaxSM.Major)
         ShaderModel = MaxSM;
@@ -423,27 +432,11 @@ std::vector<uint32_t> DXILtoSPIRV(IDXCompiler*            pLibrary,
     if (ShaderModel.Major == MaxSM.Major && ShaderModel.Minor > MaxSM.Minor)
         ShaderModel = MaxSM;
 
-    std::wstring Profile;
-    switch (ShaderCI.Desc.ShaderType)
-    {
-        // clang-format off
-        case SHADER_TYPE_VERTEX:        Profile = L"vs_"; break;
-        case SHADER_TYPE_PIXEL:         Profile = L"ps_"; break;
-        case SHADER_TYPE_GEOMETRY:      Profile = L"gs_"; break;
-        case SHADER_TYPE_HULL:          Profile = L"hs_"; break;
-        case SHADER_TYPE_DOMAIN:        Profile = L"ds_"; break;
-        case SHADER_TYPE_COMPUTE:       Profile = L"cs_"; break;
-        case SHADER_TYPE_AMPLIFICATION: Profile = L"as_"; break;
-        case SHADER_TYPE_MESH:          Profile = L"ms_"; break;
-        default: UNEXPECTED("Unexpected shader type");
-            // clang-format on
-    }
-
-    Profile += L'0' + (ShaderModel.Major % 10);
-    Profile += L'_';
-    Profile += L'0' + (ShaderModel.Minor % 10);
+    const auto         Profile = GetHLSLProfileString(ShaderCI.Desc.ShaderType, ShaderModel);
+    const std::wstring wstrProfile{Profile.begin(), Profile.end()};
+    const std::wstring wstrEntryPoint{ShaderCI.EntryPoint, ShaderCI.EntryPoint + strlen(ShaderCI.EntryPoint)};
 
-    const wchar_t* pArgs[] =
+    const wchar_t* pSpirvArgs[] =
         {
             L"-spirv",
             L"-fspv-reflect",
@@ -452,6 +445,20 @@ std::vector<uint32_t> DXILtoSPIRV(IDXCompiler*            pLibrary,
             L"-O3", // Optimization level 3
         };
 
+    const wchar_t* pDxbcArgs[] =
+        {
+            L"-Zpc", // Matrices in column-major order
+                     //L"-WX",  // Warnings as errors
+#ifdef DILIGENT_DEBUG
+            L"-Zi", // Debug info
+            //L"-Qembed_debug", // Embed debug info into the shader (some compilers do not recognize this flag)
+            L"-Od", // Disable optimization
+#else
+            L"-Od", // TODO: something goes wrong if optimization is enabled
+                    //L"-O3", // Optimization level 3
+#endif
+        };
+
     CComPtr<IDxcBlob> compiled;
     CComPtr<IDxcBlob> errors;
 
@@ -461,18 +468,17 @@ std::vector<uint32_t> DXILtoSPIRV(IDXCompiler*            pLibrary,
 
     CA.Source                     = Source.c_str();
     CA.SourceLength               = static_cast<Uint32>(Source.length());
-    auto wstrEntryPoint           = std::wstring{ShaderCI.EntryPoint, ShaderCI.EntryPoint + strlen(ShaderCI.EntryPoint)};
     CA.EntryPoint                 = wstrEntryPoint.c_str();
-    CA.Profile                    = Profile.c_str();
+    CA.Profile                    = wstrProfile.c_str();
     CA.pDefines                   = nullptr;
     CA.DefinesCount               = 0;
-    CA.pArgs                      = pArgs;
-    CA.ArgsCount                  = _countof(pArgs);
+    CA.pArgs                      = m_Target == DXCompilerTarget::Direct3D12 ? pDxbcArgs : pSpirvArgs;
+    CA.ArgsCount                  = m_Target == DXCompilerTarget::Direct3D12 ? _countof(pDxbcArgs) : _countof(pSpirvArgs);
     CA.pShaderSourceStreamFactory = ShaderCI.pShaderSourceStreamFactory;
     CA.ppBlobOut                  = &compiled;
     CA.ppCompilerOutput           = &errors;
 
-    auto result = pLibrary->Compile(CA);
+    auto result = Compile(CA);
 
     const size_t CompilerMsgLen = errors ? errors->GetBufferSize() : 0;
     const char*  CompilerMsg    = CompilerMsgLen > 0 ? static_cast<const char*>(errors->GetBufferPointer()) : nullptr;
@@ -499,15 +505,15 @@ std::vector<uint32_t> DXILtoSPIRV(IDXCompiler*            pLibrary,
         }
     }
 
-    std::vector<uint32_t> SPIRV;
     if (result && compiled && compiled->GetBufferSize() > 0)
     {
-        SPIRV.assign(static_cast<uint32_t*>(compiled->GetBufferPointer()),
-                     static_cast<uint32_t*>(compiled->GetBufferPointer()) + compiled->GetBufferSize() / sizeof(uint32_t));
+        if (pByteCode != nullptr)
+            pByteCode->assign(static_cast<uint32_t*>(compiled->GetBufferPointer()),
+                              static_cast<uint32_t*>(compiled->GetBufferPointer()) + compiled->GetBufferSize() / sizeof(uint32_t));
+
+        if (ppByteCodeBlob != nullptr)
+            *ppByteCodeBlob = compiled.Detach();
     }
-    return SPIRV;
 }
 
-#endif // VULKAN_SUPPORTED
-
 } // namespace Diligent
diff --git a/Graphics/ShaderTools/src/DXILUtilsStub.cpp b/Graphics/ShaderTools/src/DXILUtilsStub.cpp
index 4a801fc0..5cdb842a 100644
--- a/Graphics/ShaderTools/src/DXILUtilsStub.cpp
+++ b/Graphics/ShaderTools/src/DXILUtilsStub.cpp
@@ -35,12 +35,4 @@ IDXCompiler* CreateDXCompiler(DXCompilerTarget Target, const char* pLibraryName)
     return nullptr;
 }
 
-std::vector<uint32_t> DXILtoSPIRV(IDXCompiler*            pLibrary,
-                                  const ShaderCreateInfo& Attribs,
-                                  const char*             ExtraDefinitions,
-                                  IDataBlob**             ppCompilerOutput) noexcept(false)
-{
-    return {};
-}
-
 } // namespace Diligent
diff --git a/Graphics/ShaderTools/src/HLSLUtils.cpp b/Graphics/ShaderTools/src/HLSLUtils.cpp
index 94ffcd3e..613eea25 100644
--- a/Graphics/ShaderTools/src/HLSLUtils.cpp
+++ b/Graphics/ShaderTools/src/HLSLUtils.cpp
@@ -65,4 +65,30 @@ String BuildHLSLSourceString(const ShaderCreateInfo& ShaderCI,
     return HLSLSource;
 }
 
+String GetHLSLProfileString(SHADER_TYPE ShaderType, ShaderVersion ShaderModel)
+{
+    String strShaderProfile;
+    switch (ShaderType)
+    {
+        // clang-format off
+        case SHADER_TYPE_VERTEX:        strShaderProfile = "vs"; break;
+        case SHADER_TYPE_PIXEL:         strShaderProfile = "ps"; break;
+        case SHADER_TYPE_GEOMETRY:      strShaderProfile = "gs"; break;
+        case SHADER_TYPE_HULL:          strShaderProfile = "hs"; break;
+        case SHADER_TYPE_DOMAIN:        strShaderProfile = "ds"; break;
+        case SHADER_TYPE_COMPUTE:       strShaderProfile = "cs"; break;
+        case SHADER_TYPE_AMPLIFICATION: strShaderProfile = "as"; break;
+        case SHADER_TYPE_MESH:          strShaderProfile = "ms"; break;
+        // clang-format on
+        default: UNEXPECTED("Unknown shader type");
+    }
+
+    strShaderProfile += "_";
+    strShaderProfile += std::to_string(ShaderModel.Major);
+    strShaderProfile += "_";
+    strShaderProfile += std::to_string(ShaderModel.Minor);
+
+    return strShaderProfile;
+}
+
 } // namespace Diligent