Add support for specifying a subfolder under the lib/ target. Refs #750

* This allows a buidler to customise from e.g. /usr/lib/librenderdoc.so
  to /usr/lib/renderdoc/librenderdoc.so - which is harmless since the
  library is 'private' and not intended to be linked against directly.

renderdoc/CMakeLists.txt

@@ -323,7 +323,7 @@ if(ANDROID)
     set_target_properties(renderdoc PROPERTIES OUTPUT_NAME "VkLayer_GLES_RenderDoc")
 endif()
 
-install (TARGETS renderdoc DESTINATION lib${LIB_SUFFIX})
+install (TARGETS renderdoc DESTINATION lib${LIB_SUFFIX}/${LIB_SUBFOLDER})
 
 # Copy in application API header to include
 install (FILES api/app/renderdoc_app.h DESTINATION include RENAME renderdoc.h)

renderdoc/driver/vulkan/CMakeLists.txt

@@ -71,7 +71,7 @@ elseif(UNIX)
     set(json_in ${CMAKE_CURRENT_SOURCE_DIR}/renderdoc.json)
     set(json_out ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/renderdoc_capture.json)
     add_custom_command(OUTPUT ${json_out}
-        COMMAND sed '{s%\\[MAJOR\\]%${RENDERDOC_VERSION_MAJOR}%\; s%\\[MINOR\\]%${RENDERDOC_VERSION_MINOR}%\; s%...renderdoc.dll%${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX}/librenderdoc.so%}' < ${json_in} > ${json_out}
+        COMMAND sed '{s%\\[MAJOR\\]%${RENDERDOC_VERSION_MAJOR}%\; s%\\[MINOR\\]%${RENDERDOC_VERSION_MINOR}%\; s%...renderdoc.dll%${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX}/${LIB_SUBFOLDER_TRAIL_SLASH}librenderdoc.so%}' < ${json_in} > ${json_out}
         DEPENDS ${json_in})
     add_custom_target(generate-json ALL DEPENDS ${json_out})
 

renderdoc/os/posix/posix_process.cpp

@@ -498,6 +498,15 @@ uint32_t Process::LaunchAndInjectIntoProcess(const char *app, const char *workin
     FileIO::GetExecutableFilename(binpath);
     binpath = dirname(binpath);
     libpath = binpath + "/../lib";
+
+// point to the right customiseable path
+#if defined(RENDERDOC_LIB_SUFFIX)
+    libpath += STRINGIZE(RENDERDOC_LIB_SUFFIX);
+#endif
+
+#if defined(RENDERDOC_LIB_SUBFOLDER)
+    libpath += "/" STRINGIZE(RENDERDOC_LIB_SUBFOLDER);
+#endif
   }
 
   string optstr;

renderdoc/os/posix/posix_stringio.cpp

@@ -153,8 +153,17 @@ string GetReplayAppFilename()
   }
 
   // if it's not in the same directory, try in a sibling /bin
-  // e.g. /foo/bar/lib/librenderdoc.so -> /foo/bar/bin/qrenderdoc
-  replay = path + "/../bin/qrenderdoc";
+  //
+  // start from our path
+  replay = path + "/";
+
+// if there's a custom lib subfolder, go up one (e.g. /usr/lib/renderdoc/librenderdoc.so)
+#if defined(RENDERDOC_LIB_SUBFOLDER)
+  replay += "../";
+#endif
+
+  // leave the lib/ folder, and go into bin/
+  replay += "../bin/qrenderdoc";
 
   f = FileIO::fopen(replay.c_str(), "r");
   if(f)