Make xlib and xcb support optional (but default on) on linux

* Also allow disabling GL a little better by removing the dependency in
  renderdoccmd.
* Disabling them now completely removes all xcb and xlib build
  dependencies. The resulting library/executable is only useful in
  limited situations - e.g. replaying vulkan remotely which doesn't need
  any window system interaction. Or capturing vulkan as well with the
  KHR_display WSI extension which doesn't need any built-time includes
  or libs.

CMakeLists.txt

@@ -31,6 +31,9 @@ option(ENABLE_VULKAN "Enable Vulkan driver" ON)
 option(ENABLE_RENDERDOCCMD "Enable renderdoccmd" ON)
 option(ENABLE_QRENDERDOC "Enable qrenderdoc" ON)
 
+option(ENABLE_XLIB "Enable xlib windowing support" ON)
+option(ENABLE_XCB "Enable xcb windowing support" ON)
+
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
 
@@ -68,6 +71,14 @@ elseif(APPLE)
     add_definitions(-DRENDERDOC_PLATFORM_APPLE)
 elseif(UNIX)
     add_definitions(-DRENDERDOC_PLATFORM_LINUX)
+
+    if(ENABLE_XLIB)
+        add_definitions(-DRENDERDOC_WINDOWING_XLIB)
+    endif()
+
+    if(ENABLE_XCB)
+        add_definitions(-DRENDERDOC_WINDOWING_XCB)
+    endif()
 endif()
 
 add_subdirectory(renderdoc)

renderdoc/CMakeLists.txt

@@ -29,23 +29,34 @@ elseif(APPLE)
 elseif(UNIX)
     find_package(PkgConfig REQUIRED)
     find_package(Threads REQUIRED)
-    find_package(X11 REQUIRED)
-
-    # This should be fixed and moved to FindXCB.cmake
-    pkg_check_modules(PC_XCB REQUIRED xcb xcb-keysyms)
-    set(XCB_INCLUDE_DIRS)
-    set(XCB_LIBRARIES xcb xcb-keysyms)
-
-    list(APPEND RDOC_INCLUDES
-        PRIVATE ${X11_X11_INCLUDE_PATH}
-        PRIVATE ${XCB_INCLUDE_DIRS})
 
     list(APPEND RDOC_LIBRARIES
         PRIVATE m
         PRIVATE dl
-        PRIVATE ${CMAKE_THREAD_LIBS_INIT}
-        PRIVATE ${X11_X11_LIB}
-        PRIVATE ${XCB_LIBRARIES})
+        PRIVATE ${CMAKE_THREAD_LIBS_INIT})
+
+    if(ENABLE_XLIB)
+        find_package(X11 REQUIRED)
+
+        list(APPEND RDOC_INCLUDES
+            PRIVATE ${X11_X11_INCLUDE_PATH})
+
+        list(APPEND RDOC_LIBRARIES
+            PRIVATE ${X11_X11_LIB})
+    endif()
+
+    if(ENABLE_XCB)
+        # This should be fixed and moved to FindXCB.cmake
+        pkg_check_modules(PC_XCB REQUIRED xcb xcb-keysyms)
+        set(XCB_INCLUDE_DIRS)
+        set(XCB_LIBRARIES xcb xcb-keysyms)
+
+        list(APPEND RDOC_INCLUDES
+            PRIVATE ${XCB_INCLUDE_DIRS})
+
+        list(APPEND RDOC_LIBRARIES
+            PRIVATE ${XCB_LIBRARIES})
+    endif()
 endif()
 
 set(sources

renderdoc/driver/gl/gl_replay_linux.cpp

@@ -23,8 +23,6 @@
  * THE SOFTWARE.
  ******************************************************************************/
 
-#define RENDERDOC_WINDOWING_XLIB 1
-
 #include "gl_replay.h"
 #include <dlfcn.h>
 #include "gl_driver.h"
@@ -77,10 +75,16 @@ uint64_t GLReplay::MakeOutputWindow(WindowingSystem system, void *data, bool dep
 
   if(system == eWindowingSystem_Xlib)
   {
+#if defined(RENDERDOC_WINDOWING_XLIB)
     XlibWindowData *xlib = (XlibWindowData *)data;
 
     dpy = xlib->display;
     draw = xlib->window;
+#else
+    RDCERR(
+        "Xlib windowing system data passed in, but support is not compiled in. GL must have xlib "
+        "support compiled in");
+#endif
   }
   else if(system == eWindowingSystem_Unknown)
   {

renderdoc/driver/vulkan/CMakeLists.txt

@@ -46,9 +46,13 @@ if(ANDROID)
 elseif(UNIX)
     list(APPEND sources vk_posix.cpp vk_linux.cpp)
 
-    list(APPEND definitions
-        PRIVATE -DVK_USE_PLATFORM_XCB_KHR
-        PRIVATE -DVK_USE_PLATFORM_XLIB_KHR)
+    if(ENABLE_XLIB)
+        add_definitions(-DVK_USE_PLATFORM_XLIB_KHR)
+    endif()
+
+    if(ENABLE_XCB)
+        add_definitions(-DVK_USE_PLATFORM_XCB_KHR)
+    endif()
 
     set(json_in ${CMAKE_CURRENT_SOURCE_DIR}/renderdoc.json)
     set(json_out ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/renderdoc_capture.json)

renderdoc/driver/vulkan/vk_hookset_defs.h

@@ -58,7 +58,7 @@
               const VkAndroidSurfaceCreateInfoKHR *, pCreateInfo, const VkAllocationCallbacks *, \
               pAllocator, VkSurfaceKHR *, pSurface);
 
-#elif defined(VK_USE_PLATFORM_XCB_KHR) || defined(VK_USE_PLATFORM_XLIB_KHR)
+#else
 
 #if defined(VK_USE_PLATFORM_XCB_KHR)
 

renderdoc/driver/vulkan/vk_linux.cpp

@@ -22,9 +22,6 @@
  * THE SOFTWARE.
  ******************************************************************************/
 
-#define RENDERDOC_WINDOWING_XLIB 1
-#define RENDERDOC_WINDOWING_XCB 1
-
 #include "api/replay/renderdoc_replay.h"
 
 #include "vk_core.h"
@@ -34,22 +31,32 @@ void VulkanReplay::OutputWindow::SetWindowHandle(WindowingSystem system, void *d
 {
   m_WindowSystem = system;
 
+#if defined(RENDERDOC_WINDOWING_XLIB)
   if(system == eWindowingSystem_Xlib)
   {
     XlibWindowData *xdata = (XlibWindowData *)data;
     xlib.display = xdata->display;
     xlib.window = xdata->window;
+    return;
   }
-  else if(system == eWindowingSystem_XCB)
+#endif
+
+#if defined(RENDERDOC_WINDOWING_XCB)
+  if(system == eWindowingSystem_XCB)
   {
     XCBWindowData *xdata = (XCBWindowData *)data;
     xcb.connection = xdata->connection;
     xcb.window = xdata->window;
+    return;
   }
+#endif
+
+  RDCERR("Unrecognised/unsupported window system %d", system);
 }
 
 void VulkanReplay::OutputWindow::CreateSurface(VkInstance inst)
 {
+#if defined(RENDERDOC_WINDOWING_XLIB)
   if(m_WindowSystem == eWindowingSystem_Xlib)
   {
     VkXlibSurfaceCreateInfoKHR createInfo;
@@ -62,8 +69,13 @@ void VulkanReplay::OutputWindow::CreateSurface(VkInstance inst)
 
     VkResult vkr = ObjDisp(inst)->CreateXlibSurfaceKHR(Unwrap(inst), &createInfo, NULL, &surface);
     RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+    return;
   }
-  else if(m_WindowSystem == eWindowingSystem_XCB)
+#endif
+
+#if defined(RENDERDOC_WINDOWING_XCB)
+  if(m_WindowSystem == eWindowingSystem_XCB)
   {
     VkXcbSurfaceCreateInfoKHR createInfo;
 
@@ -75,7 +87,12 @@ void VulkanReplay::OutputWindow::CreateSurface(VkInstance inst)
 
     VkResult vkr = ObjDisp(inst)->CreateXcbSurfaceKHR(Unwrap(inst), &createInfo, NULL, &surface);
     RDCASSERTEQUAL(vkr, VK_SUCCESS);
+
+    return;
   }
+#endif
+
+  RDCERR("Unrecognised/unsupported window system %d", m_WindowSystem);
 }
 
 void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h)
@@ -85,6 +102,7 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h
 
   OutputWindow &outw = m_OutputWindows[id];
 
+#if defined(RENDERDOC_WINDOWING_XLIB)
   if(outw.m_WindowSystem == eWindowingSystem_Xlib)
   {
     XWindowAttributes attr = {};
@@ -92,8 +110,13 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h
 
     w = (int32_t)attr.width;
     h = (int32_t)attr.height;
+
+    return;
   }
-  else if(outw.m_WindowSystem == eWindowingSystem_XCB)
+#endif
+
+#if defined(RENDERDOC_WINDOWING_XCB)
+  if(outw.m_WindowSystem == eWindowingSystem_XCB)
   {
     xcb_get_geometry_cookie_t geomCookie =
         xcb_get_geometry(outw.xcb.connection, outw.xcb.window);    // window is a xcb_drawable_t
@@ -103,7 +126,12 @@ void VulkanReplay::GetOutputWindowDimensions(uint64_t id, int32_t &w, int32_t &h
     h = (int32_t)geom->height;
 
     free(geom);
+
+    return;
   }
+#endif
+
+  RDCERR("Unrecognised/unsupported window system %d", outw.m_WindowSystem);
 }
 
 const char *VulkanLibraryName = "libvulkan.so.1";

renderdoc/driver/vulkan/vk_posix.cpp

@@ -109,12 +109,23 @@ bool WrappedVulkan::AddRequiredExtensions(bool instance, vector<string> &extensi
     if(!oneSurfaceTypeSupported)
     {
 #if defined(VK_USE_PLATFORM_ANDROID_KHR)
+
       RDCERR("Require the '%s' extension to be present", VK_KHR_ANDROID_SURFACE_EXTENSION_NAME);
+      return false;
+
 #elif defined(VK_USE_PLATFORM_XCB_KHR) || defined(VK_USE_PLATFORM_XLIB_KHR)
+
       RDCERR("Require either the '%s' or '%s' extension to be present",
              VK_KHR_XCB_SURFACE_EXTENSION_NAME, VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
-#endif
       return false;
+
+#else
+
+      // No windowing system support compiled in - allow this to continue,
+      // but this will only work for headless replay (which is feasible on some platforms)
+      return true;
+
+#endif
     }
   }
   else if(device)

renderdoc/driver/vulkan/vk_replay.h

@@ -43,17 +43,46 @@
 
 #elif defined(RENDERDOC_PLATFORM_LINUX)
 
-#define WINDOW_HANDLE_DECL        \
-  struct                          \
-  {                               \
-    Display *display;             \
-    Drawable window;              \
-  } xlib;                         \
+#if defined(RENDERDOC_WINDOWING_XLIB)
+
+#define WINDOW_HANDLE_XLIB \
+  struct                   \
+  {                        \
+    Display *display;      \
+    Drawable window;       \
+  } xlib;
+
+#else
+
+#define WINDOW_HANDLE_XLIB \
+  struct                   \
+  {                        \
+  } xlib;
+
+#endif
+
+#if defined(RENDERDOC_WINDOWING_XCB)
+
+#define WINDOW_HANDLE_XCB         \
   struct                          \
   {                               \
     xcb_connection_t *connection; \
     xcb_window_t window;          \
   } xcb;
+
+#else
+
+#define WINDOW_HANDLE_XCB \
+  struct                  \
+  {                       \
+  } xcb;
+
+#endif
+
+#define WINDOW_HANDLE_DECL \
+  WINDOW_HANDLE_XLIB       \
+  WINDOW_HANDLE_XCB
+
 #define WINDOW_HANDLE_INIT \
   RDCEraseEl(xlib);        \
   RDCEraseEl(xcb);

renderdoc/os/posix/android/android_stringio.cpp

@@ -29,8 +29,6 @@
 
 #define LOGCAT_TAG "renderdoc"
 
-typedef int Display;
-
 namespace Keyboard
 {
 void Init()
@@ -42,10 +40,6 @@ bool PlatformHasKeyInput()
   return false;
 }
 
-void CloneDisplay(Display *dpy)
-{
-}
-
 void AddInputWindow(void *wnd)
 {
 }

renderdoc/os/posix/apple/apple_stringio.cpp

@@ -39,10 +39,6 @@ bool PlatformHasKeyInput()
   return false;
 }
 
-void CloneDisplay(Display *dpy)
-{
-}
-
 void AddInputWindow(void *wnd)
 {
 }

renderdoc/os/posix/linux/linux_stringio.cpp

@@ -22,8 +22,15 @@
  * THE SOFTWARE.
  ******************************************************************************/
 
+#if defined(RENDERDOC_WINDOWING_XLIB)
 #include <X11/Xlib.h>
 #include <X11/keysym.h>
+#endif
+
+#if defined(RENDERDOC_WINDOWING_XCB)
+#include <xcb/xcb_keysyms.h>
+#endif
+
 #include <errno.h>
 #include <iconv.h>
 #include <pwd.h>
@@ -33,7 +40,6 @@
 #include <sys/types.h>
 #include <time.h>
 #include <unistd.h>
-#include <xcb/xcb_keysyms.h>
 #include "api/app/renderdoc_app.h"
 #include "common/threading.h"
 #include "os/os_specific.h"
@@ -49,9 +55,15 @@ void Init()
 
 bool PlatformHasKeyInput()
 {
+#if defined(RENDERDOC_WINDOWING_XCB) || defined(RENDERDOC_WINDOWING_XLIB)
   return true;
+#else
+  return false;
+#endif
 }
 
+#if defined(RENDERDOC_WINDOWING_XLIB)
+
 Display *CurrentXDisplay = NULL;
 
 void CloneDisplay(Display *dpy)
@@ -62,24 +74,6 @@ void CloneDisplay(Display *dpy)
   CurrentXDisplay = XOpenDisplay(XDisplayString(dpy));
 }
 
-xcb_connection_t *connection;
-xcb_key_symbols_t *symbols;
-
-void UseConnection(xcb_connection_t *conn)
-{
-  connection = conn;
-  symbols = xcb_key_symbols_alloc(conn);
-}
-
-void AddInputWindow(void *wnd)
-{
-  // TODO check against this drawable & parent window being focused in GetKeyState
-}
-
-void RemoveInputWindow(void *wnd)
-{
-}
-
 bool GetXlibKeyState(int key)
 {
   if(CurrentXDisplay == NULL)
@@ -139,6 +133,28 @@ bool GetXlibKeyState(int key)
   return (keyByte & bitMask) != 0;
 }
 
+#else
+
+// if RENDERDOC_WINDOWING_XLIB is not defined
+
+bool GetXlibKeyState(int key)
+{
+  return false;
+}
+
+#endif
+
+#if defined(RENDERDOC_WINDOWING_XCB)
+
+xcb_connection_t *connection;
+xcb_key_symbols_t *symbols;
+
+void UseConnection(xcb_connection_t *conn)
+{
+  connection = conn;
+  symbols = xcb_key_symbols_alloc(conn);
+}
+
 bool GetXCBKeyState(int key)
 {
   if(symbols == NULL)
@@ -209,6 +225,26 @@ bool GetXCBKeyState(int key)
   return ret;
 }
 
+#else
+
+// if RENDERDOC_WINDOWING_XCB is not defined
+
+bool GetXCBKeyState(int key)
+{
+  return false;
+}
+
+#endif
+
+void AddInputWindow(void *wnd)
+{
+  // TODO check against this drawable & parent window being focused in GetKeyState
+}
+
+void RemoveInputWindow(void *wnd)
+{
+}
+
 bool GetKeyState(int key)
 {
   return GetXCBKeyState(key) || GetXlibKeyState(key);

renderdoc/replay/replay_output.cpp

@@ -23,11 +23,6 @@
  * THE SOFTWARE.
  ******************************************************************************/
 
-#if defined(RENDERDOC_PLATFORM_LINUX)
-#define RENDERDOC_WINDOWING_XLIB 1
-#define RENDERDOC_WINDOWING_XCB 1
-#endif
-
 #include "common/common.h"
 #include "maths/matrix.h"
 #include "serialise/string_utils.h"
@@ -38,10 +33,22 @@ static uint64_t GetHandle(WindowingSystem system, void *data)
 #if defined(RENDERDOC_PLATFORM_LINUX)
 
   if(system == eWindowingSystem_Xlib)
+  {
+#if defined(RENDERDOC_WINDOWING_XLIB)
     return (uint64_t)((XlibWindowData *)data)->window;
+#else
+    RDCERR("Xlib windowing system data passed in, but support is not compiled in");
+#endif
+  }
 
   if(system == eWindowingSystem_XCB)
+  {
+#if defined(RENDERDOC_WINDOWING_XCB)
     return (uint64_t)((XCBWindowData *)data)->window;
+#else
+    RDCERR("XCB windowing system data passed in, but support is not compiled in");
+#endif
+  }
 
   RDCERR("Unrecognised window system %d", system);
 

renderdoccmd/CMakeLists.txt

@@ -11,11 +11,25 @@ elseif(ANDROID)
 elseif(UNIX)
     list(APPEND sources renderdoccmd_linux.cpp)
 
-    find_package(OpenGL REQUIRED)
-    list(APPEND includes PRIVATE ${OPENGL_INCLUDE_DIR})
-    list(APPEND libraries PRIVATE ${OPENGL_gl_LIBRARY})
+    if(ENABLE_GL)
+        add_definitions(-DRENDERDOC_SUPPORT_GL)
 
-    list(APPEND libraries PRIVATE -lxcb -lX11 -lX11-xcb)
+        find_package(OpenGL REQUIRED)
+        list(APPEND includes PRIVATE ${OPENGL_INCLUDE_DIR})
+        list(APPEND libraries PRIVATE ${OPENGL_gl_LIBRARY})
+    endif()
+
+    if(ENABLE_XLIB)
+        list(APPEND libraries PRIVATE -lX11)
+    endif()
+
+    if(ENABLE_XCB)
+        list(APPEND libraries PRIVATE -lxcb)
+    endif()
+
+    if(ENABLE_XLIB AND ENABLE_XCB)
+        list(APPEND libraries PRIVATE -lX11-xcb)
+    endif()
 endif()
 
 if(ANDROID)

renderdoccmd/renderdoccmd_linux.cpp

@@ -24,7 +24,6 @@
  ******************************************************************************/
 
 #include "renderdoccmd.h"
-#include <X11/Xlib-xcb.h>
 #include <iconv.h>
 #include <locale.h>
 #include <signal.h>
@@ -32,8 +31,10 @@
 #include <unistd.h>
 #include <string>
 
-#define RENDERDOC_WINDOWING_XLIB 1
-#define RENDERDOC_WINDOWING_XCB 1
+#if defined(RENDERDOC_WINDOWING_XLIB)
+#include <X11/Xlib-xcb.h>
+#endif
+
 #include <replay/renderdoc_replay.h>
 
 using std::string;
@@ -47,6 +48,10 @@ void Daemonise()
 void DisplayRendererPreview(ReplayRenderer *renderer, TextureDisplay &displayCfg, uint32_t width,
                             uint32_t height)
 {
+// we only have the preview implemented for platforms that have xlib & xcb. It's unlikely
+// a meaningful platform exists with only one, and at the time of writing no other windowing
+// systems are supported on linux for the replay
+#if defined(RENDERDOC_WINDOWING_XLIB) && defined(RENDERDOC_WINDOWING_XCB)
   // need to create a hybrid setup xlib and xcb in case only one or the other is supported.
   // We'll prefer xcb
 
@@ -196,13 +201,20 @@ void DisplayRendererPreview(ReplayRenderer *renderer, TextureDisplay &displayCfg
 
     usleep(100000);
   }
+#else
+  std::cerr << "No supporting windowing systems defined at build time (xlib and xcb)" << std::endl;
+#endif
 }
 
+#if defined(RENDERDOC_SUPPORT_GL)
+
 // symbol defined in libGL but not librenderdoc.
 // Forces link of libGL after renderdoc (otherwise all symbols would
 // be resolved and libGL wouldn't link, meaning dlsym(RTLD_NEXT) would fai
 extern "C" void glXWaitGL();
 
+#endif
+
 void sig_handler(int signo)
 {
   if(usingKillSignal)
@@ -215,10 +227,14 @@ int main(int argc, char *argv[])
 {
   std::setlocale(LC_CTYPE, "");
 
+#if defined(RENDERDOC_SUPPORT_GL)
+
   volatile bool never_run = false;
   if(never_run)
     glXWaitGL();
 
+#endif
+
   signal(SIGINT, sig_handler);
   signal(SIGTERM, sig_handler);