Add support for specifying a subfolder under the lib/ target. Refs #750

* This allows a buidler to customise from e.g. /usr/lib/librenderdoc.so
  to /usr/lib/renderdoc/librenderdoc.so - which is harmless since the
  library is 'private' and not intended to be linked against directly.

CMakeLists.txt

@@ -40,6 +40,20 @@ set(RENDERDOC_PLUGINS_PATH "" CACHE STRING "Path to RenderDoc plugins folder aft
 set(RENDERDOC_APK_PATH "" CACHE STRING "Path to RenderDocCmd.apk after installation of RenderDoc on host (either absolute or relative to binary)")
 set(RENDERDOC_LAYER_PATH "" CACHE STRING "Path to ABI directories (i.e. lib in lib/armeabi-v7a/libVkLayer_GLES_RenderDoc.so) after installation of RenderDoc on host (either absolute or relative to dir)")
 
+set(LIB_SUFFIX "" CACHE STRING "Suffix for 'lib' folder in target directory structure. E.g. set to '64' to use /usr/local/lib64 instead of /usr/local/lib.")
+set(LIB_SUBFOLDER "" CACHE STRING "Subfolder under the 'lib' folder in target directory structure. E.g. set to 'renderdoc' to use /usr/local/lib/renderdoc instead of /usr/local/lib.")
+
+if(NOT LIB_SUFFIX STREQUAL "")
+    add_definitions(-DRENDERDOC_LIB_SUFFIX=${LIB_SUFFIX})
+endif()
+
+set(LIB_SUBFOLDER_TRAIL_SLASH "")
+
+if(NOT LIB_SUBFOLDER STREQUAL "")
+    add_definitions(-DRENDERDOC_LIB_SUBFOLDER=${LIB_SUBFOLDER})
+    set(LIB_SUBFOLDER_TRAIL_SLASH "${LIB_SUBFOLDER}/")
+endif()
+
 if(BUILD_VERSION_STABLE)
     add_definitions(-DRENDERDOC_STABLE_BUILD=1)
 endif()

qrenderdoc/CMakeLists.txt

@@ -102,6 +102,9 @@ file(WRITE
      "QMAKE_CXXFLAGS+=${QMAKE_CXXFLAGS}\n"
      "QMAKE_LFLAGS+=${QMAKE_LDFLAGS}\n"
      "\n"
+     "LIB_SUFFIX=${LIB_SUFFIX}\n"
+     "LIB_SUBFOLDER_TRAIL_SLASH=${LIB_SUBFOLDER_TRAIL_SLASH}\n"
+     "\n"
      # search for -lrenderdoc here
      "LIBS+=-L${CMAKE_RUNTIME_OUTPUT_DIRECTORY}\n"
      "\n"

qrenderdoc/Code/pyrenderdoc/CMakeLists.txt

@@ -33,7 +33,7 @@ set(MODULE_DEFINES
 # Set up rpath to find librenderdoc.so
 set(CMAKE_SKIP_BUILD_RPATH TRUE)
 set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-set(CMAKE_INSTALL_RPATH "$ORIGIN/:$ORIGIN/../lib/")
+set(CMAKE_INSTALL_RPATH "$ORIGIN/:$ORIGIN/../lib${LIB_SUFFIX}/${LIB_SUBFOLDER_TRAIL_SLASH}")
 
 # Add python library
 set (CMAKE_SHARED_LINKER_FLAGS "${PYTHON_LIBRARY} ${CMAKE_SHARED_LINKER_FLAGS}")

qrenderdoc/qrenderdoc.pro

@@ -115,7 +115,7 @@ win32 {
 
 	# Link against the core library
 	LIBS += -lrenderdoc
-	QMAKE_LFLAGS += '-Wl,-rpath,\'\$$ORIGIN\',-rpath,\'\$$ORIGIN/../lib\''
+	QMAKE_LFLAGS += '-Wl,-rpath,\'\$$ORIGIN\',-rpath,\'\$$ORIGIN/../lib'$$LIB_SUFFIX'/'$$LIB_SUBFOLDER_TRAIL_SLASH'\''
 
 	# Add the SWIG files that were generated in cmake
 	SOURCES += $$CMAKE_DIR/qrenderdoc/renderdoc_python.cxx

renderdoc/CMakeLists.txt

@@ -323,7 +323,7 @@ if(ANDROID)
     set_target_properties(renderdoc PROPERTIES OUTPUT_NAME "VkLayer_GLES_RenderDoc")
 endif()
 
-install (TARGETS renderdoc DESTINATION lib${LIB_SUFFIX})
+install (TARGETS renderdoc DESTINATION lib${LIB_SUFFIX}/${LIB_SUBFOLDER})
 
 # Copy in application API header to include
 install (FILES api/app/renderdoc_app.h DESTINATION include RENAME renderdoc.h)

renderdoc/driver/vulkan/CMakeLists.txt

@@ -71,7 +71,7 @@ elseif(UNIX)
     set(json_in ${CMAKE_CURRENT_SOURCE_DIR}/renderdoc.json)
     set(json_out ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/renderdoc_capture.json)
     add_custom_command(OUTPUT ${json_out}
-        COMMAND sed '{s%\\[MAJOR\\]%${RENDERDOC_VERSION_MAJOR}%\; s%\\[MINOR\\]%${RENDERDOC_VERSION_MINOR}%\; s%...renderdoc.dll%${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX}/librenderdoc.so%}' < ${json_in} > ${json_out}
+        COMMAND sed '{s%\\[MAJOR\\]%${RENDERDOC_VERSION_MAJOR}%\; s%\\[MINOR\\]%${RENDERDOC_VERSION_MINOR}%\; s%...renderdoc.dll%${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX}/${LIB_SUBFOLDER_TRAIL_SLASH}librenderdoc.so%}' < ${json_in} > ${json_out}
         DEPENDS ${json_in})
     add_custom_target(generate-json ALL DEPENDS ${json_out})
 

renderdoc/os/posix/posix_process.cpp

@@ -498,6 +498,15 @@ uint32_t Process::LaunchAndInjectIntoProcess(const char *app, const char *workin
     FileIO::GetExecutableFilename(binpath);
     binpath = dirname(binpath);
     libpath = binpath + "/../lib";
+
+// point to the right customiseable path
+#if defined(RENDERDOC_LIB_SUFFIX)
+    libpath += STRINGIZE(RENDERDOC_LIB_SUFFIX);
+#endif
+
+#if defined(RENDERDOC_LIB_SUBFOLDER)
+    libpath += "/" STRINGIZE(RENDERDOC_LIB_SUBFOLDER);
+#endif
   }
 
   string optstr;

renderdoc/os/posix/posix_stringio.cpp

@@ -153,8 +153,17 @@ string GetReplayAppFilename()
   }
 
   // if it's not in the same directory, try in a sibling /bin
-  // e.g. /foo/bar/lib/librenderdoc.so -> /foo/bar/bin/qrenderdoc
-  replay = path + "/../bin/qrenderdoc";
+  //
+  // start from our path
+  replay = path + "/";
+
+// if there's a custom lib subfolder, go up one (e.g. /usr/lib/renderdoc/librenderdoc.so)
+#if defined(RENDERDOC_LIB_SUBFOLDER)
+  replay += "../";
+#endif
+
+  // leave the lib/ folder, and go into bin/
+  replay += "../bin/qrenderdoc";
 
   f = FileIO::fopen(replay.c_str(), "r");
   if(f)

renderdoccmd/CMakeLists.txt

@@ -45,7 +45,7 @@ if(ANDROID)
 else()
     set(CMAKE_SKIP_BUILD_RPATH TRUE)
     set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-    set(CMAKE_INSTALL_RPATH "$ORIGIN/:$ORIGIN/../lib/")
+    set(CMAKE_INSTALL_RPATH "$ORIGIN/:$ORIGIN/../lib${LIB_SUFFIX}/${LIB_SUBFOLDER_TRAIL_SLASH}")
     set(CMAKE_EXE_LINKER_FLAGS "${LINKER_FLAGS}")
 
     add_executable(renderdoccmd ${sources})