Refactoring shader compilation - part II

9 files changed, 214 insertions, 109 deletions

diff --git a/Graphics/ShaderTools/CMakeLists.txt b/Graphics/ShaderTools/CMakeLists.txt
index 8ad4b38a..8a23ea6a 100644
--- a/Graphics/ShaderTools/CMakeLists.txt
+++ b/Graphics/ShaderTools/CMakeLists.txt
@@ -15,6 +15,11 @@ if(VULKAN_SUPPORTED OR GL_SUPPORTED OR GLES_SUPPORTED)
     list(APPEND INCLUDE include/GLSLUtils.hpp)
 endif()
 
+if(D3D11_SUPPORTED OR D3D12_SUPPORTED OR VULKAN_SUPPORTED)
+    list(APPEND SOURCE src/HLSLUtils.cpp)
+    list(APPEND INCLUDE include/HLSLUtils.hpp)
+endif()
+
 if(PLATFORM_WIN32 OR PLATFORM_UNIVERSAL_WINDOWS OR PLATFORM_LINUX)
     set(DXC_SUPPORTED TRUE)
 endif()
diff --git a/Graphics/ShaderTools/include/GLSLUtils.hpp b/Graphics/ShaderTools/include/GLSLUtils.hpp
index 115155eb..4c8b92c8 100644
--- a/Graphics/ShaderTools/include/GLSLUtils.hpp
+++ b/Graphics/ShaderTools/include/GLSLUtils.hpp
@@ -40,7 +40,7 @@ enum TargetGLSLCompiler
     driver
 };
 
-String BuildGLSLSourceString(const ShaderCreateInfo& CreationAttribs,
+String BuildGLSLSourceString(const ShaderCreateInfo& ShaderCI,
                              const DeviceCaps&       deviceCaps,
                              TargetGLSLCompiler      TargetCompiler,
                              const char*             ExtraDefinitions = nullptr);
diff --git a/Graphics/ShaderTools/include/HLSLUtils.hpp b/Graphics/ShaderTools/include/HLSLUtils.hpp
new file mode 100644
index 00000000..8d8774f5
--- /dev/null
+++ b/Graphics/ShaderTools/include/HLSLUtils.hpp
@@ -0,0 +1,40 @@
+/*
+ *  Copyright 2019-2020 Diligent Graphics LLC
+ *  Copyright 2015-2019 Egor Yusov
+ *  
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  In no event and under no legal theory, whether in tort (including negligence), 
+ *  contract, or otherwise, unless required by applicable law (such as deliberate 
+ *  and grossly negligent acts) or agreed to in writing, shall any Contributor be
+ *  liable for any damages, including any direct, indirect, special, incidental, 
+ *  or consequential damages of any character arising as a result of this License or 
+ *  out of the use or inability to use the software (including but not limited to damages 
+ *  for loss of goodwill, work stoppage, computer failure or malfunction, or any and 
+ *  all other commercial damages or losses), even if such Contributor has been advised 
+ *  of the possibility of such damages.
+ */
+
+#pragma once
+
+#include "BasicTypes.h"
+#include "GraphicsTypes.h"
+#include "Shader.h"
+
+namespace Diligent
+{
+
+String BuildHLSLSourceString(const ShaderCreateInfo& ShaderCI,
+                             const char*             ExtraDefinitions = nullptr);
+
+} // namespace Diligent
diff --git a/Graphics/ShaderTools/include/ShaderToolsCommon.hpp b/Graphics/ShaderTools/include/ShaderToolsCommon.hpp
index 4108d319..f75b9fe7 100644
--- a/Graphics/ShaderTools/include/ShaderToolsCommon.hpp
+++ b/Graphics/ShaderTools/include/ShaderToolsCommon.hpp
@@ -29,6 +29,8 @@
 
 #include "GraphicsTypes.h"
 #include "Shader.h"
+#include "RefCntAutoPtr.hpp"
+#include "DataBlob.h"
 
 namespace Diligent
 {
@@ -65,4 +67,16 @@ void AppendShaderMacros(std::string& Source, const ShaderMacro* Macros);
 /// etc.
 void AppendShaderTypeDefinitions(std::string& Source, SHADER_TYPE Type);
 
+
+/// Reads shader source code from a file or uses the one from the shader create info
+const char* ReadShaderSourceFile(const char*                      SourceCode,
+                                 IShaderSourceInputStreamFactory* pShaderSourceStreamFactory,
+                                 const char*                      FilePath,
+                                 RefCntAutoPtr<IDataBlob>&        pFileData,
+                                 size_t&                          SourceCodeLen) noexcept(false);
+
+/// Appends shader source code to the source string
+void AppendShaderSourceCode(std::string& Source, const ShaderCreateInfo& ShaderCI) noexcept(false);
+
+
 } // namespace Diligent
diff --git a/Graphics/ShaderTools/src/DXCompiler.cpp b/Graphics/ShaderTools/src/DXCompiler.cpp
index 322574a2..80235f65 100644
--- a/Graphics/ShaderTools/src/DXCompiler.cpp
+++ b/Graphics/ShaderTools/src/DXCompiler.cpp
@@ -48,6 +48,8 @@
 #    include <d3d12shader.h>
 #endif
 
+#include "HLSLUtils.hpp"
+
 namespace Diligent
 {
 
@@ -403,67 +405,16 @@ void DXCompilerImpl::GetD3D12ShaderReflection(IDxcBlob*                pShaderBy
 
 
 #if VULKAN_SUPPORTED
-// Implemented in GLSLSourceBuilder.cpp
-const char* GetShaderTypeDefines(SHADER_TYPE Type);
-
-namespace
-{
-
-// clang-format off
-static const char g_HLSLDefinitions[] =
-{
-#include "../../GraphicsEngineD3DBase/include/HLSLDefinitions_inc.fxh"
-};
-// clang-format on
-
-} // namespace
 
 std::vector<uint32_t> DXILtoSPIRV(IDXCompiler*            pLibrary,
-                                  const ShaderCreateInfo& Attribs,
+                                  const ShaderCreateInfo& ShaderCI,
                                   const char*             ExtraDefinitions,
                                   IDataBlob**             ppCompilerOutput) noexcept(false)
 {
-    RefCntAutoPtr<IDataBlob> pFileData{MakeNewRCObj<DataBlobImpl>()(0)};
 
-    const char* SourceCode    = 0;
-    int         SourceCodeLen = 0;
-    if (Attribs.Source)
-    {
-        SourceCode    = Attribs.Source;
-        SourceCodeLen = static_cast<int>(strlen(Attribs.Source));
-    }
-    else
-    {
-        VERIFY(Attribs.pShaderSourceStreamFactory, "Input stream factory is null");
-        RefCntAutoPtr<IFileStream> pSourceStream;
-        Attribs.pShaderSourceStreamFactory->CreateInputStream(Attribs.FilePath, &pSourceStream);
-        if (pSourceStream == nullptr)
-            LOG_ERROR_AND_THROW("Failed to open shader source file");
-
-        pSourceStream->ReadBlob(pFileData);
-        SourceCode    = reinterpret_cast<char*>(pFileData->GetDataPtr());
-        SourceCodeLen = static_cast<int>(pFileData->GetSize());
-    }
-
-    std::string Source;
-    Source.reserve(SourceCodeLen + sizeof(g_HLSLDefinitions));
-
-    Source.append(g_HLSLDefinitions);
-    AppendShaderTypeDefinitions(Source, Attribs.Desc.ShaderType);
-
-    if (ExtraDefinitions != nullptr)
-        Source += ExtraDefinitions;
-
-    if (Attribs.Macros != nullptr)
-    {
-        Source += '\n';
-        AppendShaderMacros(Source, Attribs.Macros);
-    }
-
-    Source.append(SourceCode, SourceCodeLen);
 
     // validate shader version
-    ShaderVersion ShaderModel = Attribs.HLSLVersion;
+    ShaderVersion ShaderModel = ShaderCI.HLSLVersion;
     ShaderVersion MaxSM       = pLibrary->GetMaxShaderModel();
 
     if (ShaderModel.Major < 6 || ShaderModel.Major > MaxSM.Major)
@@ -473,7 +424,7 @@ std::vector<uint32_t> DXILtoSPIRV(IDXCompiler*            pLibrary,
         ShaderModel = MaxSM;
 
     std::wstring Profile;
-    switch (Attribs.Desc.ShaderType)
+    switch (ShaderCI.Desc.ShaderType)
     {
         // clang-format off
         case SHADER_TYPE_VERTEX:        Profile = L"vs_"; break;
@@ -506,16 +457,18 @@ std::vector<uint32_t> DXILtoSPIRV(IDXCompiler*            pLibrary,
 
     IDXCompiler::CompileAttribs CA;
 
+    auto Source = BuildHLSLSourceString(ShaderCI, ExtraDefinitions);
+
     CA.Source                     = Source.c_str();
     CA.SourceLength               = static_cast<Uint32>(Source.length());
-    auto wstrEntryPoint           = std::wstring{Attribs.EntryPoint, Attribs.EntryPoint + strlen(Attribs.EntryPoint)};
+    auto wstrEntryPoint           = std::wstring{ShaderCI.EntryPoint, ShaderCI.EntryPoint + strlen(ShaderCI.EntryPoint)};
     CA.EntryPoint                 = wstrEntryPoint.c_str();
     CA.Profile                    = Profile.c_str();
     CA.pDefines                   = nullptr;
     CA.DefinesCount               = 0;
     CA.pArgs                      = pArgs;
     CA.ArgsCount                  = _countof(pArgs);
-    CA.pShaderSourceStreamFactory = Attribs.pShaderSourceStreamFactory;
+    CA.pShaderSourceStreamFactory = ShaderCI.pShaderSourceStreamFactory;
     CA.ppBlobOut                  = &compiled;
     CA.ppCompilerOutput           = &errors;
 
@@ -538,11 +491,11 @@ std::vector<uint32_t> DXILtoSPIRV(IDXCompiler*            pLibrary,
     {
         if (ppCompilerOutput != nullptr)
         {
-            LOG_ERROR_AND_THROW("Failed to compile Vulkan shader \"", (Attribs.Desc.Name != nullptr ? Attribs.Desc.Name : ""), "\".");
+            LOG_ERROR_AND_THROW("Failed to compile Vulkan shader \"", (ShaderCI.Desc.Name != nullptr ? ShaderCI.Desc.Name : ""), "\".");
         }
         else
         {
-            LOG_ERROR_AND_THROW("Failed to compile Vukan shader \"", (Attribs.Desc.Name != nullptr ? Attribs.Desc.Name : ""), "\":\n", (CompilerMsg != nullptr ? std::string(CompilerMsg, CompilerMsgLen) : "<no compiler log available>"));
+            LOG_ERROR_AND_THROW("Failed to compile Vukan shader \"", (ShaderCI.Desc.Name != nullptr ? ShaderCI.Desc.Name : ""), "\":\n", (CompilerMsg != nullptr ? std::string(CompilerMsg, CompilerMsgLen) : "<no compiler log available>"));
         }
     }
 
diff --git a/Graphics/ShaderTools/src/GLSLUtils.cpp b/Graphics/ShaderTools/src/GLSLUtils.cpp
index 0d8ef9f4..86451a87 100644
--- a/Graphics/ShaderTools/src/GLSLUtils.cpp
+++ b/Graphics/ShaderTools/src/GLSLUtils.cpp
@@ -265,26 +265,10 @@ String BuildGLSLSourceString(const ShaderCreateInfo& ShaderCI,
         AppendShaderMacros(GLSLSource, ShaderCI.Macros);
     }
 
-    RefCntAutoPtr<IDataBlob> pFileData(MakeNewRCObj<DataBlobImpl>()(0));
+    RefCntAutoPtr<IDataBlob> pFileData;
+    size_t                   SourceLen = 0;
 
-    auto   ShaderSource = ShaderCI.Source;
-    size_t SourceLen    = 0;
-    if (ShaderSource)
-    {
-        SourceLen = strlen(ShaderSource);
-    }
-    else
-    {
-        VERIFY(ShaderCI.pShaderSourceStreamFactory, "Input stream factory is null");
-        RefCntAutoPtr<IFileStream> pSourceStream;
-        ShaderCI.pShaderSourceStreamFactory->CreateInputStream(ShaderCI.FilePath, &pSourceStream);
-        if (pSourceStream == nullptr)
-            LOG_ERROR_AND_THROW("Failed to open shader source file");
-
-        pSourceStream->ReadBlob(pFileData);
-        ShaderSource = reinterpret_cast<char*>(pFileData->GetDataPtr());
-        SourceLen    = pFileData->GetSize();
-    }
+    const auto* ShaderSource = ReadShaderSourceFile(ShaderCI.Source, ShaderCI.pShaderSourceStreamFactory, ShaderCI.FilePath, pFileData, SourceLen);
 
     if (ShaderCI.SourceLanguage == SHADER_SOURCE_LANGUAGE_HLSL)
     {
@@ -319,7 +303,9 @@ String BuildGLSLSourceString(const ShaderCreateInfo& ShaderCI,
 #endif
     }
     else
+    {
         GLSLSource.append(ShaderSource, SourceLen);
+    }
 
     return GLSLSource;
 }
diff --git a/Graphics/ShaderTools/src/GLSLangUtils.cpp b/Graphics/ShaderTools/src/GLSLangUtils.cpp
index 98aad2d1..79d5edeb 100644
--- a/Graphics/ShaderTools/src/GLSLangUtils.cpp
+++ b/Graphics/ShaderTools/src/GLSLangUtils.cpp
@@ -398,7 +398,7 @@ public:
         m_pInputStreamFactory->CreateInputStream(headerName, &pSourceStream);
         if (pSourceStream == nullptr)
         {
-            LOG_ERROR("Failed to open shader include file \"", headerName, "\". Check that the file exists");
+            LOG_ERROR("Failed to open shader include file '", headerName, "'. Check that the file exists");
             return nullptr;
         }
 
@@ -439,64 +439,47 @@ private:
     std::unordered_map<IncludeResult*, RefCntAutoPtr<IDataBlob>> m_DataBlobs;
 };
 
-std::vector<unsigned int> HLSLtoSPIRV(const ShaderCreateInfo& Attribs,
+std::vector<unsigned int> HLSLtoSPIRV(const ShaderCreateInfo& ShaderCI,
                                       const char*             ExtraDefinitions,
                                       IDataBlob**             ppCompilerOutput)
 {
-    EShLanguage        ShLang = ShaderTypeToShLanguage(Attribs.Desc.ShaderType);
+    EShLanguage        ShLang = ShaderTypeToShLanguage(ShaderCI.Desc.ShaderType);
     ::glslang::TShader Shader{ShLang};
     EShMessages        messages = (EShMessages)(EShMsgSpvRules | EShMsgVulkanRules | EShMsgReadHlsl | EShMsgHlslLegalization);
 
-    VERIFY_EXPR(Attribs.SourceLanguage == SHADER_SOURCE_LANGUAGE_HLSL);
+    VERIFY_EXPR(ShaderCI.SourceLanguage == SHADER_SOURCE_LANGUAGE_HLSL);
 
     Shader.setEnvInput(::glslang::EShSourceHlsl, ShLang, ::glslang::EShClientVulkan, 100);
     Shader.setEnvClient(::glslang::EShClientVulkan, ::glslang::EShTargetVulkan_1_0);
     Shader.setEnvTarget(::glslang::EShTargetSpv, ::glslang::EShTargetSpv_1_0);
     Shader.setHlslIoMapping(true);
-    Shader.setEntryPoint(Attribs.EntryPoint);
+    Shader.setEntryPoint(ShaderCI.EntryPoint);
     Shader.setEnvTargetHlslFunctionality1();
 
-    RefCntAutoPtr<IDataBlob> pFileData(MakeNewRCObj<DataBlobImpl>()(0));
+    RefCntAutoPtr<IDataBlob> pFileData;
+    size_t                   SourceCodeLen = 0;
 
-    const char* SourceCode    = 0;
-    int         SourceCodeLen = 0;
-    if (Attribs.Source)
-    {
-        SourceCode    = Attribs.Source;
-        SourceCodeLen = static_cast<int>(strlen(Attribs.Source));
-    }
-    else
-    {
-        VERIFY(Attribs.pShaderSourceStreamFactory, "Input stream factory is null");
-        RefCntAutoPtr<IFileStream> pSourceStream;
-        Attribs.pShaderSourceStreamFactory->CreateInputStream(Attribs.FilePath, &pSourceStream);
-        if (pSourceStream == nullptr)
-            LOG_ERROR_AND_THROW("Failed to open shader source file");
-
-        pSourceStream->ReadBlob(pFileData);
-        SourceCode    = reinterpret_cast<char*>(pFileData->GetDataPtr());
-        SourceCodeLen = static_cast<int>(pFileData->GetSize());
-    }
+    const char* SourceCode = ReadShaderSourceFile(ShaderCI.Source, ShaderCI.pShaderSourceStreamFactory, ShaderCI.FilePath, pFileData, SourceCodeLen);
 
     std::string Defines = g_HLSLDefinitions;
-    AppendShaderTypeDefinitions(Defines, Attribs.Desc.ShaderType);
+    AppendShaderTypeDefinitions(Defines, ShaderCI.Desc.ShaderType);
 
     if (ExtraDefinitions != nullptr)
         Defines += ExtraDefinitions;
 
-    if (Attribs.Macros != nullptr)
+    if (ShaderCI.Macros != nullptr)
     {
         Defines += '\n';
-        AppendShaderMacros(Defines, Attribs.Macros);
+        AppendShaderMacros(Defines, ShaderCI.Macros);
     }
     Shader.setPreamble(Defines.c_str());
 
     const char* ShaderStrings[]       = {SourceCode};
-    const int   ShaderStringLenghts[] = {SourceCodeLen};
-    const char* Names[]               = {Attribs.FilePath != nullptr ? Attribs.FilePath : ""};
+    const int   ShaderStringLenghts[] = {static_cast<int>(SourceCodeLen)};
+    const char* Names[]               = {ShaderCI.FilePath != nullptr ? ShaderCI.FilePath : ""};
     Shader.setStringsWithLengthsAndNames(ShaderStrings, ShaderStringLenghts, Names, 1);
 
-    IncluderImpl Includer(Attribs.pShaderSourceStreamFactory);
+    IncluderImpl Includer{ShaderCI.pShaderSourceStreamFactory};
 
     auto SPIRV = CompileShaderInternal(Shader, messages, &Includer, SourceCode, SourceCodeLen, ppCompilerOutput);
     if (SPIRV.empty())
diff --git a/Graphics/ShaderTools/src/HLSLUtils.cpp b/Graphics/ShaderTools/src/HLSLUtils.cpp
new file mode 100644
index 00000000..94ffcd3e
--- /dev/null
+++ b/Graphics/ShaderTools/src/HLSLUtils.cpp
@@ -0,0 +1,68 @@
+/*
+ *  Copyright 2019-2020 Diligent Graphics LLC
+ *  Copyright 2015-2019 Egor Yusov
+ *  
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *  In no event and under no legal theory, whether in tort (including negligence), 
+ *  contract, or otherwise, unless required by applicable law (such as deliberate 
+ *  and grossly negligent acts) or agreed to in writing, shall any Contributor be
+ *  liable for any damages, including any direct, indirect, special, incidental, 
+ *  or consequential damages of any character arising as a result of this License or 
+ *  out of the use or inability to use the software (including but not limited to damages 
+ *  for loss of goodwill, work stoppage, computer failure or malfunction, or any and 
+ *  all other commercial damages or losses), even if such Contributor has been advised 
+ *  of the possibility of such damages.
+ */
+
+#include <cstring>
+#include <sstream>
+
+#include "HLSLUtils.hpp"
+#include "DebugUtilities.hpp"
+#include "ShaderToolsCommon.hpp"
+
+namespace Diligent
+{
+
+// clang-format off
+static const char g_HLSLDefinitions[] =
+{
+#include "../../GraphicsEngineD3DBase/include/HLSLDefinitions_inc.fxh"
+};
+// clang-format on
+
+
+String BuildHLSLSourceString(const ShaderCreateInfo& ShaderCI,
+                             const char*             ExtraDefinitions)
+{
+    String HLSLSource;
+
+    HLSLSource.append(g_HLSLDefinitions);
+    AppendShaderTypeDefinitions(HLSLSource, ShaderCI.Desc.ShaderType);
+
+    if (ExtraDefinitions != nullptr)
+        HLSLSource += ExtraDefinitions;
+
+    if (ShaderCI.Macros != nullptr)
+    {
+        HLSLSource += '\n';
+        AppendShaderMacros(HLSLSource, ShaderCI.Macros);
+    }
+
+    AppendShaderSourceCode(HLSLSource, ShaderCI);
+
+    return HLSLSource;
+}
+
+} // namespace Diligent
diff --git a/Graphics/ShaderTools/src/ShaderToolsCommon.cpp b/Graphics/ShaderTools/src/ShaderToolsCommon.cpp
index dee4011b..dfb12bec 100644
--- a/Graphics/ShaderTools/src/ShaderToolsCommon.cpp
+++ b/Graphics/ShaderTools/src/ShaderToolsCommon.cpp
@@ -27,6 +27,7 @@
 
 #include "ShaderToolsCommon.hpp"
 #include "DebugUtilities.hpp"
+#include "DataBlobImpl.hpp"
 
 namespace Diligent
 {
@@ -85,4 +86,59 @@ void AppendShaderTypeDefinitions(std::string& Source, SHADER_TYPE Type)
     AppendShaderMacros(Source, GetShaderTypeMacros(Type));
 }
 
+
+const char* ReadShaderSourceFile(const char*                      SourceCode,
+                                 IShaderSourceInputStreamFactory* pShaderSourceStreamFactory,
+                                 const char*                      FilePath,
+                                 RefCntAutoPtr<IDataBlob>&        pFileData,
+                                 size_t&                          SourceCodeLen) noexcept(false)
+{
+    SourceCodeLen = 0;
+    if (SourceCode != nullptr)
+    {
+        VERIFY(pShaderSourceStreamFactory == nullptr, "pShaderSourceStreamFactory must be null when SourceCode is not null");
+        SourceCodeLen = strlen(SourceCode);
+    }
+    else
+    {
+        if (pShaderSourceStreamFactory != nullptr)
+        {
+            if (FilePath != nullptr)
+            {
+                RefCntAutoPtr<IFileStream> pSourceStream;
+                pShaderSourceStreamFactory->CreateInputStream(FilePath, &pSourceStream);
+                if (pSourceStream == nullptr)
+                    LOG_ERROR_AND_THROW("Failed to load shader source file '", FilePath, '\'');
+
+                pFileData = MakeNewRCObj<DataBlobImpl>{}(0);
+                pSourceStream->ReadBlob(pFileData);
+                SourceCode    = reinterpret_cast<char*>(pFileData->GetDataPtr());
+                SourceCodeLen = pFileData->GetSize();
+            }
+            else
+            {
+                UNEXPECTED("FilePath is null");
+            }
+        }
+        else
+        {
+            UNEXPECTED("Input stream factory is null");
+        }
+    }
+
+    return SourceCode;
+}
+
+void AppendShaderSourceCode(std::string& Source, const ShaderCreateInfo& ShaderCI) noexcept(false)
+{
+    RefCntAutoPtr<IDataBlob> pFileData;
+
+    size_t SourceCodeLen = 0;
+
+    const auto* SourceCode =
+        ReadShaderSourceFile(ShaderCI.Source, ShaderCI.pShaderSourceStreamFactory,
+                             ShaderCI.FilePath, pFileData, SourceCodeLen);
+    Source.append(SourceCode, SourceCodeLen);
+}
+
 } // namespace Diligent