Enabled XLib support in Vulkan on Linux (fixed https://github.com/DiligentGraphics/DiligentEngine/issues/85)

3 files changed, 80 insertions, 29 deletions

diff --git a/Graphics/GraphicsEngineVulkan/CMakeLists.txt b/Graphics/GraphicsEngineVulkan/CMakeLists.txt
index 2bfd9acd..61c80fbb 100644
--- a/Graphics/GraphicsEngineVulkan/CMakeLists.txt
+++ b/Graphics/GraphicsEngineVulkan/CMakeLists.txt
@@ -257,7 +257,7 @@ endif()
 if(PLATFORM_WIN32)
     set(PRIVATE_COMPILE_DEFINITIONS VK_USE_PLATFORM_WIN32_KHR=1 NOMINMAX)
 elseif(PLATFORM_LINUX)
-    set(PRIVATE_COMPILE_DEFINITIONS VK_USE_PLATFORM_XCB_KHR=1)
+    set(PRIVATE_COMPILE_DEFINITIONS VK_USE_PLATFORM_XCB_KHR=1 VK_USE_PLATFORM_XLIB_KHR=1)
 elseif(PLATFORM_MACOS)
     set(PRIVATE_COMPILE_DEFINITIONS VK_USE_PLATFORM_MACOS_MVK=1)
 elseif(PLATFORM_IOS)
diff --git a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanHeaders.h b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanHeaders.h
index 6794a4ee..98bad1e8 100644
--- a/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanHeaders.h
+++ b/Graphics/GraphicsEngineVulkan/include/VulkanUtilities/VulkanHeaders.h
@@ -43,3 +43,18 @@
 #if USE_VOLK
 #    include "volk/volk.h"
 #endif
+
+#if defined(VK_USE_PLATFORM_XLIB_KHR) || defined(_X11_XLIB_H_)
+
+// Undef symbols defined by XLib
+#    ifdef Bool
+#        undef Bool
+#    endif
+#    ifdef True
+#        undef True
+#    endif
+#    ifdef False
+#        undef False
+#    endif
+
+#endif
diff --git a/Graphics/GraphicsEngineVulkan/src/SwapChainVkImpl.cpp b/Graphics/GraphicsEngineVulkan/src/SwapChainVkImpl.cpp
index 1789596f..ab53d639 100644
--- a/Graphics/GraphicsEngineVulkan/src/SwapChainVkImpl.cpp
+++ b/Graphics/GraphicsEngineVulkan/src/SwapChainVkImpl.cpp
@@ -68,51 +68,87 @@ void SwapChainVkImpl::CreateSurface()
     }
 
     // Create OS-specific surface
+    VkResult err = VK_ERROR_INITIALIZATION_FAILED;
 #if defined(VK_USE_PLATFORM_WIN32_KHR)
-    VkWin32SurfaceCreateInfoKHR surfaceCreateInfo = {};
+    if (m_Window.hWnd != NULL)
+    {
+        VkWin32SurfaceCreateInfoKHR surfaceCreateInfo = {};
 
-    surfaceCreateInfo.sType     = VK_STRUCTURE_TYPE_WIN32_SURFACE_CREATE_INFO_KHR;
-    surfaceCreateInfo.hinstance = GetModuleHandle(NULL);
-    surfaceCreateInfo.hwnd      = (HWND)m_Window.hWnd;
+        surfaceCreateInfo.sType     = VK_STRUCTURE_TYPE_WIN32_SURFACE_CREATE_INFO_KHR;
+        surfaceCreateInfo.hinstance = GetModuleHandle(NULL);
+        surfaceCreateInfo.hwnd      = (HWND)m_Window.hWnd;
 
-    auto err = vkCreateWin32SurfaceKHR(m_VulkanInstance->GetVkInstance(), &surfaceCreateInfo, nullptr, &m_VkSurface);
+        err = vkCreateWin32SurfaceKHR(m_VulkanInstance->GetVkInstance(), &surfaceCreateInfo, nullptr, &m_VkSurface);
+    }
 #elif defined(VK_USE_PLATFORM_ANDROID_KHR)
-    VkAndroidSurfaceCreateInfoKHR surfaceCreateInfo = {};
+    if (m_Window.pAWindow != nullptr)
+    {
+        VkAndroidSurfaceCreateInfoKHR surfaceCreateInfo = {};
 
-    surfaceCreateInfo.sType  = VK_STRUCTURE_TYPE_ANDROID_SURFACE_CREATE_INFO_KHR;
-    surfaceCreateInfo.window = (ANativeWindow*)m_Window.pAWindow;
+        surfaceCreateInfo.sType  = VK_STRUCTURE_TYPE_ANDROID_SURFACE_CREATE_INFO_KHR;
+        surfaceCreateInfo.window = (ANativeWindow*)m_Window.pAWindow;
 
-    auto err = vkCreateAndroidSurfaceKHR(m_VulkanInstance->GetVkInstance(), &surfaceCreateInfo, NULL, &m_VkSurface);
+        err = vkCreateAndroidSurfaceKHR(m_VulkanInstance->GetVkInstance(), &surfaceCreateInfo, NULL, &m_VkSurface);
+    }
 #elif defined(VK_USE_PLATFORM_IOS_MVK)
-    VkIOSSurfaceCreateInfoMVK surfaceCreateInfo = {};
+    if (m_Window.pCALayer != nullptr)
+    {
+        VkIOSSurfaceCreateInfoMVK surfaceCreateInfo = {};
 
-    surfaceCreateInfo.sType = VK_STRUCTURE_TYPE_IOS_SURFACE_CREATE_INFO_MVK;
-    surfaceCreateInfo.pView = m_Window.pCALayer;
+        surfaceCreateInfo.sType = VK_STRUCTURE_TYPE_IOS_SURFACE_CREATE_INFO_MVK;
+        surfaceCreateInfo.pView = m_Window.pCALayer;
 
-    auto err = vkCreateIOSSurfaceMVK(m_VulkanInstance->GetVkInstance(), &surfaceCreateInfo, nullptr, &m_VkSurface);
+        err = vkCreateIOSSurfaceMVK(m_VulkanInstance->GetVkInstance(), &surfaceCreateInfo, nullptr, &m_VkSurface);
+    }
 #elif defined(VK_USE_PLATFORM_MACOS_MVK)
-    VkMacOSSurfaceCreateInfoMVK surfaceCreateInfo = {};
+    if (m_Window.pNSView != nullptr)
+    {
+        VkMacOSSurfaceCreateInfoMVK surfaceCreateInfo = {};
 
-    surfaceCreateInfo.sType = VK_STRUCTURE_TYPE_MACOS_SURFACE_CREATE_INFO_MVK;
-    surfaceCreateInfo.pView = m_Window.pNSView;
+        surfaceCreateInfo.sType = VK_STRUCTURE_TYPE_MACOS_SURFACE_CREATE_INFO_MVK;
+        surfaceCreateInfo.pView = m_Window.pNSView;
 
-    auto err = vkCreateMacOSSurfaceMVK(m_VulkanInstance->GetVkInstance(), &surfaceCreateInfo, NULL, &m_VkSurface);
+        err = vkCreateMacOSSurfaceMVK(m_VulkanInstance->GetVkInstance(), &surfaceCreateInfo, NULL, &m_VkSurface);
+    }
 #elif defined(VK_USE_PLATFORM_WAYLAND_KHR)
-    VkWaylandSurfaceCreateInfoKHR surfaceCreateInfo = {};
+    if (m_Window.pDisplay != nullptr)
+    {
+        VkWaylandSurfaceCreateInfoKHR surfaceCreateInfo = {};
 
-    surfaceCreateInfo.sType   = VK_STRUCTURE_TYPE_WAYLAND_SURFACE_CREATE_INFO_KHR;
-    surfaceCreateInfo.display = reinterpret_cast<struct wl_display*>(m_Window.pDisplay);
-    surfaceCreateInfo.Surface = reinterpret_cast<struct wl_surface*>(nullptr);
+        surfaceCreateInfo.sType   = VK_STRUCTURE_TYPE_WAYLAND_SURFACE_CREATE_INFO_KHR;
+        surfaceCreateInfo.display = reinterpret_cast<struct wl_display*>(m_Window.pDisplay);
+        surfaceCreateInfo.Surface = reinterpret_cast<struct wl_surface*>(nullptr);
 
-    err = vkCreateWaylandSurfaceKHR(m_VulkanInstance->GetVkInstance(), &surfaceCreateInfo, nullptr, &m_VkSurface);
-#elif defined(VK_USE_PLATFORM_XCB_KHR)
-    VkXcbSurfaceCreateInfoKHR surfaceCreateInfo = {};
+        err = vkCreateWaylandSurfaceKHR(m_VulkanInstance->GetVkInstance(), &surfaceCreateInfo, nullptr, &m_VkSurface);
+    }
+#elif defined(VK_USE_PLATFORM_XCB_KHR) || defined(VK_USE_PLATFORM_XLIB_KHR)
 
-    surfaceCreateInfo.sType      = VK_STRUCTURE_TYPE_XCB_SURFACE_CREATE_INFO_KHR;
-    surfaceCreateInfo.connection = reinterpret_cast<xcb_connection_t*>(m_Window.pXCBConnection);
-    surfaceCreateInfo.window     = m_Window.WindowId;
+#    if defined(VK_USE_PLATFORM_XCB_KHR)
+    if (m_Window.pXCBConnection != nullptr && m_Window.WindowId != 0)
+    {
+        VkXcbSurfaceCreateInfoKHR surfaceCreateInfo = {};
+
+        surfaceCreateInfo.sType      = VK_STRUCTURE_TYPE_XCB_SURFACE_CREATE_INFO_KHR;
+        surfaceCreateInfo.connection = reinterpret_cast<xcb_connection_t*>(m_Window.pXCBConnection);
+        surfaceCreateInfo.window     = m_Window.WindowId;
+
+        err = vkCreateXcbSurfaceKHR(m_VulkanInstance->GetVkInstance(), &surfaceCreateInfo, nullptr, &m_VkSurface);
+    }
+#    endif
+
+#    if defined(VK_USE_PLATFORM_XLIB_KHR)
+    if ((m_Window.pDisplay != nullptr && m_Window.WindowId != 0) && m_VkSurface == VK_NULL_HANDLE)
+    {
+        VkXlibSurfaceCreateInfoKHR surfaceCreateInfo = {};
+
+        surfaceCreateInfo.sType  = VK_STRUCTURE_TYPE_XLIB_SURFACE_CREATE_INFO_KHR;
+        surfaceCreateInfo.dpy    = reinterpret_cast<Display*>(m_Window.pDisplay);
+        surfaceCreateInfo.window = m_Window.WindowId;
+
+        err = vkCreateXlibSurfaceKHR(m_VulkanInstance->GetVkInstance(), &surfaceCreateInfo, nullptr, &m_VkSurface);
+    }
+#    endif
 
-    auto err = vkCreateXcbSurfaceKHR(m_VulkanInstance->GetVkInstance(), &surfaceCreateInfo, nullptr, &m_VkSurface);
 #endif
 
     CHECK_VK_ERROR_AND_THROW(err, "Failed to create OS-specific surface");