diff --git a/CMakeLists.txt b/CMakeLists.txt index 433b50618..3fb243d2b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,6 +99,16 @@ set(LIB_SUFFIX "" CACHE STRING "Suffix for 'lib' folder in target directory stru set(LIB_SUBFOLDER "" CACHE STRING "Subfolder under the 'lib' folder in target directory structure. E.g. set to 'renderdoc' to use /usr/local/lib/renderdoc instead of /usr/local/lib.") set(VULKAN_JSON_SUFFIX "" CACHE STRING "Suffix for the vulkan implicit_layer json file. E.g. set to '.x86_64' to use renderdoc_capture.x86_64.json instead of renderdoc_capture.json") +option(INTERNAL_SELF_CAPTURE "Internal option: enable self-capture" OFF) + +set(RDOC_BASE_NAME "renderdoc") +if(INTERNAL_SELF_CAPTURE) + set(RDOC_BASE_NAME "rdocself") + add_definitions(-DRDOC_SELFCAPTURE_LIMITEDAPI=1) +endif() +string(TOUPPER ${RDOC_BASE_NAME} RDOC_BASE_NAME_UPPER) +add_definitions(-DRDOC_BASE_NAME=${RDOC_BASE_NAME}) + if(NOT LIB_SUFFIX STREQUAL "") add_definitions(-DRENDERDOC_LIB_SUFFIX=${LIB_SUFFIX}) endif() @@ -263,6 +273,16 @@ if(APPLE) set(ENABLE_EGL OFF CACHE BOOL "" FORCE) endif() +if(INTERNAL_SELF_CAPTURE) + message(STATUS "Disabling GL/GLES for self-capture") + set(ENABLE_GLES OFF CACHE BOOL "" FORCE) + set(ENABLE_EGL OFF CACHE BOOL "" FORCE) + + message(STATUS "Disabling qrenderdoc & python modules for self-capture") + set(ENABLE_QRENDERDOC OFF CACHE BOOL "" FORCE) + set(ENABLE_PYRENDERDOC OFF CACHE BOOL "" FORCE) +endif() + if(ANDROID) if(ENABLE_GL) message(STATUS "Disabling GL driver on android") @@ -522,3 +542,7 @@ if(UNIX AND NOT ANDROID AND NOT APPLE AND NOT ENABLE_GGP) endif() endif() +if(INTERNAL_SELF_CAPTURE) + message(STATUS "INTERNAL: Configuring for self-capture") +endif() + diff --git a/qrenderdoc/Windows/MainWindow.cpp b/qrenderdoc/Windows/MainWindow.cpp index 61557bf00..082336cc8 100644 --- a/qrenderdoc/Windows/MainWindow.cpp +++ b/qrenderdoc/Windows/MainWindow.cpp @@ -394,7 +394,12 @@ MainWindow::MainWindow(ICaptureContext &ctx) : QMainWindow(NULL), ui(new Ui::Mai ui->action_Recompress_Capture->setEnabled(false); #if defined(Q_OS_WIN32) - if(GetModuleHandleA("rdocself.dll")) +#define SELF_HOST_NAME "rdocself.dll" +#else +#define SELF_HOST_NAME "librdocself.so" +#endif + + if(RENDERDOC_CanSelfHostedCapture(SELF_HOST_NAME)) { QAction *begin = new QAction(tr("Start Self-hosted Capture"), this); QAction *end = new QAction(tr("End Self-hosted Capture"), this); @@ -404,21 +409,20 @@ MainWindow::MainWindow(ICaptureContext &ctx) : QMainWindow(NULL), ui(new Ui::Mai begin->setEnabled(false); end->setEnabled(true); - RENDERDOC_StartSelfHostCapture("rdocself.dll"); + RENDERDOC_StartSelfHostCapture(SELF_HOST_NAME); }); QObject::connect(end, &QAction::triggered, [begin, end]() { begin->setEnabled(true); end->setEnabled(false); - RENDERDOC_EndSelfHostCapture("rdocself.dll"); + RENDERDOC_EndSelfHostCapture(SELF_HOST_NAME); }); ui->menu_Tools->addSeparator(); ui->menu_Tools->addAction(begin); ui->menu_Tools->addAction(end); } -#endif m_Ctx.AddCaptureViewer(this); diff --git a/renderdoc/CMakeLists.txt b/renderdoc/CMakeLists.txt index 925e26a2d..693f86313 100644 --- a/renderdoc/CMakeLists.txt +++ b/renderdoc/CMakeLists.txt @@ -609,7 +609,7 @@ target_link_libraries(renderdoc ${RDOC_LIBRARIES}) add_dependencies(renderdoc renderdoc_libentry) if(UNIX AND NOT ANDROID AND NOT APPLE) - set(RDOC_LINK_FLAGS "-Wl,--undefined,force_include_libentry -Wl,--version-script,${CMAKE_CURRENT_SOURCE_DIR}/renderdoc.version") + set(RDOC_LINK_FLAGS "-Wl,--undefined,force_include_libentry -Wl,--version-script,${CMAKE_CURRENT_SOURCE_DIR}/${RDOC_BASE_NAME}.version") if(NOT ENABLE_ASAN AND NOT ENABLE_TSAN) set(RDOC_LINK_FLAGS "${RDOC_LINK_FLAGS} -Wl,--no-undefined") @@ -636,6 +636,10 @@ if(APPLE) set_target_properties(renderdoc PROPERTIES BUILD_WITH_INSTALL_NAME_DIR TRUE) endif() +if(INTERNAL_SELF_CAPTURE) + set_target_properties(renderdoc PROPERTIES OUTPUT_NAME "rdocself") +endif() + if(ANDROID) set(RDOC_LINK_FLAGS "-Wl,--undefined,force_include_libentry -Wl,--build-id") if(ENABLE_ASAN) diff --git a/renderdoc/api/replay/renderdoc_replay.h b/renderdoc/api/replay/renderdoc_replay.h index fbb63b45c..bbac3949f 100644 --- a/renderdoc/api/replay/renderdoc_replay.h +++ b/renderdoc/api/replay/renderdoc_replay.h @@ -30,6 +30,21 @@ #include "apidefs.h" +#if defined(RDOC_SELFCAPTURE_LIMITEDAPI) + +#define RENDERDOC_AllocArrayMem RDOCSELF_AllocArrayMem +#define RENDERDOC_FreeArrayMem RDOCSELF_FreeArrayMem +#define RENDERDOC_GetDefaultCaptureOptions RDOCSELF_GetDefaultCaptureOptions +#define RENDERDOC_NeedVulkanLayerRegistration RDOCSELF_NeedVulkanLayerRegistration +#define RENDERDOC_UpdateVulkanLayerRegistration RDOCSELF_UpdateVulkanLayerRegistration +#define RENDERDOC_ExecuteAndInject RDOCSELF_ExecuteAndInject +#define RENDERDOC_InjectIntoProcess RDOCSELF_InjectIntoProcess +#define RENDERDOC_GetCommitHash RDOCSELF_GetCommitHash +#define RENDERDOC_InitialiseReplay RDOCSELF_InitialiseReplay +#define RENDERDOC_ShutdownReplay RDOCSELF_ShutdownReplay + +#endif + // this #define can be used to mark a program as a 'replay' program which should not be captured. // Any program used for such purpose must define and export this symbol in the main exe or one dll // that will be loaded before renderdoc.dll is loaded. @@ -1986,6 +2001,13 @@ extern "C" RENDERDOC_API ExecuteResult RENDERDOC_CC RENDERDOC_InjectIntoProcess(uint32_t pid, const rdcarray &env, const rdcstr &capturefile, const CaptureOptions &opts, bool waitForExit); +DOCUMENT(R"(When debugging RenderDoc it can be useful to capture itself by doing a side-build with a +temporary name. This function checks to see if a given self-hosted DLL is available. + +:param str dllname: The name of the self-hosted capture module. +)"); +extern "C" RENDERDOC_API bool RENDERDOC_CC RENDERDOC_CanSelfHostedCapture(const rdcstr &dllname); + DOCUMENT(R"(When debugging RenderDoc it can be useful to capture itself by doing a side-build with a temporary name. This function wraps up the use of the in-application API to start a capture. diff --git a/renderdoc/core/core.cpp b/renderdoc/core/core.cpp index d66097fc4..f29e23491 100644 --- a/renderdoc/core/core.cpp +++ b/renderdoc/core/core.cpp @@ -46,6 +46,8 @@ #include "replay/renderdoc_serialise.inl" +extern "C" const rdcstr VulkanLayerJSONBasename = STRINGIZE(RDOC_BASE_NAME); + RDOC_DEBUG_CONFIG(bool, Capture_Debug_SnapshotDiagnosticLog, false, "Snapshot the diagnostic log at capture time and embed in the capture."); @@ -601,6 +603,14 @@ void RenderDoc::RemoveHooks() void RenderDoc::InitialiseReplay(GlobalEnvironment env, const rdcarray &args) { + if(!IsReplayApp()) + { + RDCERR( + "Initialising replay within non-replaying app. Did you properly export replay marker in " + "host executable or library, or are you trying to replay directly with a self-hosted " + "capture build?"); + } + m_GlobalEnv = env; #if ENABLED(RDOC_LINUX) && ENABLED(RDOC_XLIB) diff --git a/renderdoc/driver/vulkan/CMakeLists.txt b/renderdoc/driver/vulkan/CMakeLists.txt index 9f6816f49..8d5672af9 100644 --- a/renderdoc/driver/vulkan/CMakeLists.txt +++ b/renderdoc/driver/vulkan/CMakeLists.txt @@ -100,11 +100,11 @@ elseif(ENABLE_GGP) list(APPEND sources vk_posix.cpp vk_ggp.cpp) add_definitions(-DVK_USE_PLATFORM_GGP) - set(VULKAN_LAYER_MODULE_PATH "${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX}/${LIB_SUBFOLDER_TRAIL_SLASH}librenderdoc.so") - set(VULKAN_ENABLE_VAR "ENABLE_VULKAN_RENDERDOC_CAPTURE") + set(VULKAN_LAYER_MODULE_PATH "${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX}/${LIB_SUBFOLDER_TRAIL_SLASH}lib${RDOC_BASE_NAME}.so") + set(VULKAN_ENABLE_VAR "ENABLE_VULKAN_${RDOC_BASE_NAME_UPPER}_CAPTURE") set(json_in ${CMAKE_CURRENT_SOURCE_DIR}/renderdoc.json) - set(json_out ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/renderdoc_capture.json) + set(json_out ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${RDOC_BASE_NAME}_capture.json) configure_file(${json_in} ${json_out}) @@ -126,11 +126,11 @@ elseif(UNIX) add_definitions(-DVK_USE_PLATFORM_WIN32_KHR) - set(VULKAN_LAYER_MODULE_PATH "${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX}/${LIB_SUBFOLDER_TRAIL_SLASH}librenderdoc.so") - set(VULKAN_ENABLE_VAR "ENABLE_VULKAN_RENDERDOC_CAPTURE") + set(VULKAN_LAYER_MODULE_PATH "${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX}/${LIB_SUBFOLDER_TRAIL_SLASH}lib${RDOC_BASE_NAME}.so") + set(VULKAN_ENABLE_VAR "ENABLE_VULKAN_${RDOC_BASE_NAME_UPPER}_CAPTURE") set(json_in ${CMAKE_CURRENT_SOURCE_DIR}/renderdoc.json) - set(json_out ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/renderdoc_capture${VULKAN_JSON_SUFFIX}.json) + set(json_out ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${RDOC_BASE_NAME}_capture${VULKAN_JSON_SUFFIX}.json) configure_file(${json_in} ${json_out}) diff --git a/renderdoc/driver/vulkan/vk_layer.cpp b/renderdoc/driver/vulkan/vk_layer.cpp index fecd05aa6..69e3f3356 100644 --- a/renderdoc/driver/vulkan/vk_layer.cpp +++ b/renderdoc/driver/vulkan/vk_layer.cpp @@ -36,6 +36,8 @@ #include "vk_hookset_defs.h" #include "vk_resources.h" +extern "C" const rdcstr VulkanLayerJSONBasename; + // this was removed from the vulkan definition header #undef VK_LAYER_EXPORT #define VK_LAYER_EXPORT @@ -152,18 +154,13 @@ class VulkanHook : LibraryHook Process::RegisterEnvironmentModification( EnvironmentModification(EnvMod::Set, EnvSep::NoSep, "DISABLE_LAYER", "1")); -#if ENABLED(RDOC_WIN32) - // on windows support self-hosted capture by checking our filename and tweaking the env var we - // set - rdcstr module_name; - FileIO::GetLibraryFilename(module_name); - module_name = strupper(strip_extension(get_basename(module_name))); - if(module_name != "RENDERDOC") + // support self-hosted capture by checking our filename and tweaking the env var we set + if(VulkanLayerJSONBasename != "renderdoc") { Process::RegisterEnvironmentModification(EnvironmentModification( - EnvMod::Set, EnvSep::NoSep, "ENABLE_VULKAN_" + module_name + "_CAPTURE", "1")); + EnvMod::Set, EnvSep::NoSep, + "ENABLE_VULKAN_" + strupper(VulkanLayerJSONBasename) + "_CAPTURE", "1")); } -#endif // check options to set further variables, and apply OptionsUpdated(); diff --git a/renderdoc/driver/vulkan/vk_posix.cpp b/renderdoc/driver/vulkan/vk_posix.cpp index 32fed985e..d8e34aea9 100644 --- a/renderdoc/driver/vulkan/vk_posix.cpp +++ b/renderdoc/driver/vulkan/vk_posix.cpp @@ -34,6 +34,8 @@ #include #include +extern "C" const rdcstr VulkanLayerJSONBasename; + bool VulkanReplay::IsOutputWindowVisible(uint64_t id) { if(id == 0 || m_OutputWindows.find(id) == m_OutputWindows.end()) @@ -290,7 +292,7 @@ static rdcstr GenerateJSON(const rdcstr &sopath) idx = json.find(enableVarString); while(idx >= 0) { - json = json.substr(0, idx) + STRINGIZE(ENABLE_VULKAN_RENDERDOC_CAPTURE) + + json = json.substr(0, idx) + "ENABLE_VULKAN_" + strupper(VulkanLayerJSONBasename) + "_CAPTURE" + json.substr(idx + sizeof(enableVarString) - 1); idx = json.find(enableVarString); @@ -363,24 +365,21 @@ ITERABLE_OPERATORS(LayerPath); rdcstr LayerRegistrationPath(LayerPath path) { + const rdcstr json_filename = + VulkanLayerJSONBasename + "_capture" STRINGIZE(RENDERDOC_VULKAN_JSON_SUFFIX) ".json"; + switch(path) { - case LayerPath::usr: - return "/usr/share/vulkan/implicit_layer.d/renderdoc_capture" STRINGIZE( - RENDERDOC_VULKAN_JSON_SUFFIX) ".json"; - case LayerPath::etc: - return "/etc/vulkan/implicit_layer.d/renderdoc_capture" STRINGIZE( - RENDERDOC_VULKAN_JSON_SUFFIX) ".json"; + case LayerPath::usr: return "/usr/share/vulkan/implicit_layer.d/" + json_filename; + case LayerPath::etc: return "/etc/vulkan/implicit_layer.d/" + json_filename; case LayerPath::home: { rdcstr xdg = Process::GetEnvVariable("XDG_DATA_HOME"); if(!xdg.empty() && FileIO::exists(xdg)) - return xdg + "/vulkan/implicit_layer.d/renderdoc_capture" STRINGIZE( - RENDERDOC_VULKAN_JSON_SUFFIX) ".json"; + return xdg + "/vulkan/implicit_layer.d/" + json_filename; rdcstr home_path = Process::GetEnvVariable("HOME"); - return home_path + "/.local/share/vulkan/implicit_layer.d/renderdoc_capture" STRINGIZE( - RENDERDOC_VULKAN_JSON_SUFFIX) ".json"; + return home_path + "/.local/share/vulkan/implicit_layer.d/" + json_filename; } default: break; } diff --git a/renderdoc/driver/vulkan/vk_win32.cpp b/renderdoc/driver/vulkan/vk_win32.cpp index 59903b3b2..8fdd9c5be 100644 --- a/renderdoc/driver/vulkan/vk_win32.cpp +++ b/renderdoc/driver/vulkan/vk_win32.cpp @@ -29,6 +29,7 @@ #include static int dllLocator = 0; +extern "C" const rdcstr VulkanLayerJSONBasename; void VulkanReplay::OutputWindow::SetWindowHandle(WindowingData window) { @@ -257,6 +258,8 @@ bool ProcessImplicitLayersKey(HKEY key, const rdcstr &path, rdcarray *ot for(size_t i = 0; i < myJSON.length(); i++) myJSON[i] = towlower(myJSON[i]); + rdcwstr VulkanLayerJSONFilename = StringFormat::UTF82Wide(VulkanLayerJSONBasename + ".json"); + while(ret == ERROR_SUCCESS) { // convert the name here so we preserve casing @@ -269,7 +272,7 @@ bool ProcessImplicitLayersKey(HKEY key, const rdcstr &path, rdcarray *ot { thisRegistered = true; } - else if(wcsstr(name, L"renderdoc.json") != NULL) + else if(wcsstr(name, VulkanLayerJSONFilename.c_str()) != NULL) { if(otherJSONs) otherJSONs->push_back(utf8name); diff --git a/renderdoc/os/posix/ggp/ggp_callstack.cpp b/renderdoc/os/posix/ggp/ggp_callstack.cpp index 002ef0a58..46d1abbf5 100644 --- a/renderdoc/os/posix/ggp/ggp_callstack.cpp +++ b/renderdoc/os/posix/ggp/ggp_callstack.cpp @@ -93,7 +93,7 @@ void Init() char line[512] = {0}; if(fgets(line, 511, f)) { - if(strstr(line, "librenderdoc") && strstr(line, "r-xp")) + if(strstr(line, "lib" STRINGIZE(RDOC_BASE_NAME)) && strstr(line, "r-xp")) { sscanf(line, "%p-%p", &renderdocBase, &renderdocEnd); break; diff --git a/renderdoc/os/posix/linux/linux_callstack.cpp b/renderdoc/os/posix/linux/linux_callstack.cpp index f31c6fc2d..e97d235b9 100644 --- a/renderdoc/os/posix/linux/linux_callstack.cpp +++ b/renderdoc/os/posix/linux/linux_callstack.cpp @@ -103,7 +103,7 @@ void Init() char line[512] = {0}; if(fgets(line, 511, f)) { - if(strstr(line, "librenderdoc") && strstr(line, "r-xp")) + if(strstr(line, "lib" STRINGIZE(RDOC_BASE_NAME)) && strstr(line, "r-xp")) { sscanf(line, "%p-%p", &renderdocBase, &renderdocEnd); break; diff --git a/renderdoc/os/posix/linux/linux_hook.cpp b/renderdoc/os/posix/linux/linux_hook.cpp index ef36c5e5d..7138904d8 100644 --- a/renderdoc/os/posix/linux/linux_hook.cpp +++ b/renderdoc/os/posix/linux/linux_hook.cpp @@ -544,7 +544,7 @@ void *intercept_dlopen(const char *filename, int flag, void *ret) if(cb) cb(ret); - ret = realdlopen("librenderdoc.so", flag); + ret = realdlopen("lib" STRINGIZE(RDOC_BASE_NAME) ".so", flag); break; } } diff --git a/renderdoc/os/posix/linux/linux_process.cpp b/renderdoc/os/posix/linux/linux_process.cpp index 61e56d4ee..9757d9e6d 100644 --- a/renderdoc/os/posix/linux/linux_process.cpp +++ b/renderdoc/os/posix/linux/linux_process.cpp @@ -713,7 +713,8 @@ void CacheDebuggerPresent() split(tracermaps, lines, '\n'); // remove any lines that don't reference librenderdoc.so - lines.removeIf([](const rdcstr &l) { return !l.contains("/librenderdoc.so"); }); + lines.removeIf( + [](const rdcstr &l) { return !l.contains("/lib" STRINGIZE(RDOC_BASE_NAME) ".so"); }); merge(lines, tracermaps, '\n'); if(tracermaps.contains("r-x")) diff --git a/renderdoc/os/posix/linux/linux_stringio.cpp b/renderdoc/os/posix/linux/linux_stringio.cpp index 0626b2fc1..53091aef8 100644 --- a/renderdoc/os/posix/linux/linux_stringio.cpp +++ b/renderdoc/os/posix/linux/linux_stringio.cpp @@ -650,7 +650,7 @@ void GetLibraryFilename(rdcstr &selfName) ::fclose(f); - char *c = strstr(map_string, "/librenderdoc.so"); + char *c = strstr(map_string, "/lib" STRINGIZE(RDOC_BASE_NAME) ".so"); if(c) { @@ -718,7 +718,8 @@ void GetLibraryFilename(rdcstr &selfName) if(librenderdoc_path.empty()) { - RDCWARN("Couldn't get librenderdoc.so path from /proc/self/maps, falling back to dladdr"); + RDCWARN("Couldn't get lib" STRINGIZE( + RDOC_BASE_NAME) ".so path from /proc/self/maps, falling back to dladdr"); Dl_info info; if(dladdr(&LibraryLocator, &info)) diff --git a/renderdoc/os/posix/posix_libentry.cpp b/renderdoc/os/posix/posix_libentry.cpp index fc9b2edc2..4ee7d9ac7 100644 --- a/renderdoc/os/posix/posix_libentry.cpp +++ b/renderdoc/os/posix/posix_libentry.cpp @@ -31,7 +31,7 @@ void ResetHookingEnvVars(); // DllMain equivalent void library_loaded() { - if(LibraryHooks::Detect("renderdoc__replay__marker")) + if(LibraryHooks::Detect(STRINGIZE(RDOC_BASE_NAME) "__replay__marker")) { RDCDEBUG("Not creating hooks - in replay app"); diff --git a/renderdoc/os/posix/posix_process.cpp b/renderdoc/os/posix/posix_process.cpp index facaa5c85..6baca1269 100644 --- a/renderdoc/os/posix/posix_process.cpp +++ b/renderdoc/os/posix/posix_process.cpp @@ -780,7 +780,7 @@ void GetHookingEnvMods(rdcarray &modifications, const C FileIO::GetLibraryFilename(ownlibpath); ownlibpath = get_dirname(ownlibpath); - rdcstr libfile = "librenderdoc" LIB_SUFFIX; + rdcstr libfile = "lib" STRINGIZE(RDOC_BASE_NAME) LIB_SUFFIX; // on macOS, the path must be absolute #if ENABLED(RDOC_APPLE) diff --git a/renderdoc/os/win32/win32_callstack.cpp b/renderdoc/os/win32/win32_callstack.cpp index ccc7d45c5..44483ddd2 100644 --- a/renderdoc/os/win32/win32_callstack.cpp +++ b/renderdoc/os/win32/win32_callstack.cpp @@ -483,7 +483,7 @@ static bool InitDbgHelp() else { wchar_t path[MAX_PATH] = {0}; - GetModuleFileNameW(GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"), path, MAX_PATH - 1); + GetModuleFileNameW(GetModuleHandleA(STRINGIZE(RDOC_BASE_NAME) ".dll"), path, MAX_PATH - 1); wchar_t *slash = wcsrchr(path, '\\'); diff --git a/renderdoc/os/win32/win32_hook.cpp b/renderdoc/os/win32/win32_hook.cpp index d224d426c..cf87369ba 100644 --- a/renderdoc/os/win32/win32_hook.cpp +++ b/renderdoc/os/win32/win32_hook.cpp @@ -187,7 +187,7 @@ struct CachedHookData // Also we exclude ourselves here - just in case the application has already loaded // renderdoc.dll, or tries to load it. if(strstr(lowername, "fraps") || strstr(lowername, "gameoverlayrenderer") || - strstr(lowername, STRINGIZE(RDOC_DLL_FILE) ".dll") == lowername) + strstr(lowername, STRINGIZE(RDOC_BASE_NAME) ".dll") == lowername) return; // set module pointer if we are hooking exports from this module diff --git a/renderdoc/os/win32/win32_libentry.cpp b/renderdoc/os/win32/win32_libentry.cpp index 22f4a40de..0226674d6 100644 --- a/renderdoc/os/win32/win32_libentry.cpp +++ b/renderdoc/os/win32/win32_libentry.cpp @@ -50,7 +50,7 @@ static BOOL add_hooks() } // search for an exported symbol with this name, typically renderdoc__replay__marker - if(LibraryHooks::Detect(STRINGIZE(RDOC_DLL_FILE) "__replay__marker")) + if(LibraryHooks::Detect(STRINGIZE(RDOC_BASE_NAME) "__replay__marker")) { RDCDEBUG("Not creating hooks - in replay app"); diff --git a/renderdoc/os/win32/win32_process.cpp b/renderdoc/os/win32/win32_process.cpp index f9656894f..de943a1f8 100644 --- a/renderdoc/os/win32/win32_process.cpp +++ b/renderdoc/os/win32/win32_process.cpp @@ -409,7 +409,7 @@ void InjectFunctionCall(HANDLE hProcess, uintptr_t renderdoc_remote, const char RDCDEBUG("Injecting call to %s", funcName); - HMODULE renderdoc_local = GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"); + HMODULE renderdoc_local = GetModuleHandleA(STRINGIZE(RDOC_BASE_NAME) ".dll"); uintptr_t func_local = (uintptr_t)GetProcAddress(renderdoc_local, funcName); @@ -610,7 +610,7 @@ rdcpair Process::InjectIntoProcess(uint32_t pid, RDCLOG("Injecting renderdoc into process %lu", pid); wchar_t renderdocPath[MAX_PATH] = {0}; - GetModuleFileNameW(GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"), &renderdocPath[0], + GetModuleFileNameW(GetModuleHandleA(STRINGIZE(RDOC_BASE_NAME) ".dll"), &renderdocPath[0], MAX_PATH - 1); wchar_t renderdocPathLower[MAX_PATH] = {0}; @@ -972,9 +972,9 @@ rdcpair Process::InjectIntoProcess(uint32_t pid, InjectDLL(hProcess, renderdocPath); - const char *rdoc_dll = STRINGIZE(RDOC_DLL_FILE); + const char *rdoc_dll = STRINGIZE(RDOC_BASE_NAME); - uintptr_t loc = FindRemoteDLL(pid, STRINGIZE(RDOC_DLL_FILE) ".dll"); + uintptr_t loc = FindRemoteDLL(pid, STRINGIZE(RDOC_BASE_NAME) ".dll"); rdcpair result = {ResultCode::Succeeded, 0}; @@ -1125,11 +1125,11 @@ rdcpair Process::LaunchAndInjectIntoProcess( const CaptureOptions &opts, bool waitForExit) { void *func = - GetProcAddress(GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"), "INTERNAL_SetCaptureFile"); + GetProcAddress(GetModuleHandleA(STRINGIZE(RDOC_BASE_NAME) ".dll"), "INTERNAL_SetCaptureFile"); if(func == NULL) { - const char *rdoc_dll = STRINGIZE(RDOC_DLL_FILE); + const char *rdoc_dll = STRINGIZE(RDOC_BASE_NAME); RDResult result; SET_ERROR_RESULT(result, ResultCode::InternalError, "Can't find required export function in %s.dll - corrupted/missing file?", diff --git a/renderdoc/os/win32/win32_stringio.cpp b/renderdoc/os/win32/win32_stringio.cpp index e2056a38d..7ca37e7d9 100644 --- a/renderdoc/os/win32/win32_stringio.cpp +++ b/renderdoc/os/win32/win32_stringio.cpp @@ -174,7 +174,7 @@ void GetExecutableFilename(rdcstr &selfName) void GetLibraryFilename(rdcstr &selfName) { wchar_t curFile[512] = {0}; - GetModuleFileNameW(GetModuleHandleA(STRINGIZE(RDOC_DLL_FILE) ".dll"), curFile, 511); + GetModuleFileNameW(GetModuleHandleA(STRINGIZE(RDOC_BASE_NAME) ".dll"), curFile, 511); selfName = StringFormat::Wide2UTF8(curFile); } diff --git a/renderdoc/rdocself.version b/renderdoc/rdocself.version new file mode 100644 index 000000000..dce1cd79e --- /dev/null +++ b/renderdoc/rdocself.version @@ -0,0 +1,34 @@ +{ + global: + _init; + _fini; + gl[A-Z]*; + egl[A-Z]*; + vk_icd*; + dlopen; + dlsym; + fork; + execl; + execlp; + execle; + execlpe; + execv; + execvp; + execve; + execvpe; + _exit; + VK_LAYER_RENDERDOC_*; + RENDERDOC_GetAPI; + RDOCSELF_AllocArrayMem; + RDOCSELF_FreeArrayMem; + RDOCSELF_GetDefaultCaptureOptions; + RDOCSELF_NeedVulkanLayerRegistration; + RDOCSELF_UpdateVulkanLayerRegistration; + RDOCSELF_ExecuteAndInject; + RDOCSELF_InjectIntoProcess; + RDOCSELF_GetCommitHash; + RDOCSELF_InitialiseReplay; + RDOCSELF_ShutdownReplay; + local: + *; +}; diff --git a/renderdoc/renderdoc.vcxproj b/renderdoc/renderdoc.vcxproj index 401fceb04..538551b7d 100644 --- a/renderdoc/renderdoc.vcxproj +++ b/renderdoc/renderdoc.vcxproj @@ -71,7 +71,7 @@ false ProgramDatabase $(SolutionDir)renderdoc\;$(SolutionDir)renderdoc\3rdparty\ - RDOC_DLL_FILE=$(ProjectName);RENDERDOC_EXPORTS;RENDERDOC_PLATFORM_WIN32;WIN32;NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) + RDOC_BASE_NAME=$(ProjectName);RENDERDOC_EXPORTS;RENDERDOC_PLATFORM_WIN32;WIN32;NDEBUG;_WINDOWS;_USRDLL;%(PreprocessorDefinitions) Level4 true 4100 diff --git a/renderdoc/replay/entry_points.cpp b/renderdoc/replay/entry_points.cpp index 1aed52e1b..41c7c75f9 100644 --- a/renderdoc/replay/entry_points.cpp +++ b/renderdoc/replay/entry_points.cpp @@ -532,6 +532,11 @@ extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_BecomeRemoteServer( killReplay, previewWindow); } +extern "C" RENDERDOC_API bool RENDERDOC_CC RENDERDOC_CanSelfHostedCapture(const rdcstr &dllname) +{ + return Process::IsModuleLoaded(dllname); +} + extern "C" RENDERDOC_API void RENDERDOC_CC RENDERDOC_StartSelfHostCapture(const rdcstr &dllname) { if(!Process::IsModuleLoaded(dllname)) diff --git a/renderdoccmd/CMakeLists.txt b/renderdoccmd/CMakeLists.txt index b5babd1ff..1ccc82a1e 100644 --- a/renderdoccmd/CMakeLists.txt +++ b/renderdoccmd/CMakeLists.txt @@ -86,6 +86,10 @@ else() set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${LINKER_FLAGS}") add_executable(renderdoccmd ${sources}) + + if(INTERNAL_SELF_CAPTURE) + set_target_properties(renderdoccmd PROPERTIES OUTPUT_NAME "rdocselfcmd") + endif() endif() if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") diff --git a/renderdoccmd/renderdoccmd.cpp b/renderdoccmd/renderdoccmd.cpp index 71f01dcc2..73d02858b 100644 --- a/renderdoccmd/renderdoccmd.cpp +++ b/renderdoccmd/renderdoccmd.cpp @@ -174,127 +174,6 @@ struct HelpCommand : public Command } }; -struct ThumbCommand : public Command -{ -private: - std::string infile; - std::string outfile; - std::string format; - uint32_t maxsize = 0; - -public: - ThumbCommand() : Command() {} - virtual void AddOptions(cmdline::parser &parser) - { - parser.set_footer(""); - parser.add("out", 'o', "The output filename to save the file to", true, - "filename.jpg"); - parser.add("format", 'f', - "The format of the output file. If empty, detected from filename", - false, "", cmdline::oneof("jpg", "png", "bmp", "tga")); - parser.add( - "max-size", 's', - "The maximum dimension of the thumbnail. Default is 0, which is unlimited.", false, 0); - } - virtual const char *Description() { return "Saves a capture's embedded thumbnail to disk."; } - virtual bool IsInternalOnly() { return false; } - virtual bool IsCaptureCommand() { return false; } - virtual bool Parse(cmdline::parser &parser, GlobalEnvironment &) - { - std::vector rest = parser.rest(); - if(rest.empty()) - { - std::cerr << "Error: thumb command requires a capture filename." << std::endl - << std::endl - << parser.usage(); - return false; - } - - infile = rest[0]; - - rest.erase(rest.begin()); - - parser.set_rest(rest); - - outfile = parser.get("out"); - format = parser.get("format"); - maxsize = parser.get("max-size"); - - return true; - } - - virtual int Execute(const CaptureOptions &) - { - FileType type = FileType::JPG; - - if(format == "png") - { - type = FileType::PNG; - } - else if(format == "tga") - { - type = FileType::TGA; - } - else if(format == "bmp") - { - type = FileType::BMP; - } - else - { - const char *dot = strrchr(outfile.c_str(), '.'); - - if(dot != NULL && strstr(dot, "png")) - type = FileType::PNG; - else if(dot != NULL && strstr(dot, "tga")) - type = FileType::TGA; - else if(dot != NULL && strstr(dot, "bmp")) - type = FileType::BMP; - else if(dot != NULL && strstr(dot, "jpg")) - type = FileType::JPG; - else - std::cerr << "Couldn't guess format from '" << outfile << "', defaulting to jpg." - << std::endl; - } - - bytebuf buf; - - ICaptureFile *file = RENDERDOC_OpenCaptureFile(); - ResultDetails st = file->OpenFile(conv(infile), "rdc", NULL); - if(st.OK()) - { - buf = file->GetThumbnail(type, maxsize).data; - } - else - { - std::cerr << "Couldn't open '" << infile << "': " << st.Message() << std::endl; - } - file->Shutdown(); - - if(buf.empty()) - { - std::cerr << "Couldn't fetch the thumbnail in '" << infile << "'" << std::endl; - } - else - { - FILE *f = fopen(outfile.c_str(), "wb"); - - if(!f) - { - std::cerr << "Couldn't open destination file '" << outfile << "'" << std::endl; - } - else - { - fwrite(buf.data(), 1, buf.size(), f); - fclose(f); - - std::cout << "Wrote thumbnail from '" << infile << "' to '" << outfile << "'." << std::endl; - } - } - - return 0; - } -}; - struct CaptureCommand : public Command { private: @@ -450,6 +329,129 @@ public: } }; +#if !defined(RDOC_SELFCAPTURE_LIMITEDAPI) + +struct ThumbCommand : public Command +{ +private: + std::string infile; + std::string outfile; + std::string format; + uint32_t maxsize = 0; + +public: + ThumbCommand() : Command() {} + virtual void AddOptions(cmdline::parser &parser) + { + parser.set_footer(""); + parser.add("out", 'o', "The output filename to save the file to", true, + "filename.jpg"); + parser.add("format", 'f', + "The format of the output file. If empty, detected from filename", + false, "", cmdline::oneof("jpg", "png", "bmp", "tga")); + parser.add( + "max-size", 's', + "The maximum dimension of the thumbnail. Default is 0, which is unlimited.", false, 0); + } + virtual const char *Description() { return "Saves a capture's embedded thumbnail to disk."; } + virtual bool IsInternalOnly() { return false; } + virtual bool IsCaptureCommand() { return false; } + virtual bool Parse(cmdline::parser &parser, GlobalEnvironment &) + { + std::vector rest = parser.rest(); + if(rest.empty()) + { + std::cerr << "Error: thumb command requires a capture filename." << std::endl + << std::endl + << parser.usage(); + return false; + } + + infile = rest[0]; + + rest.erase(rest.begin()); + + parser.set_rest(rest); + + outfile = parser.get("out"); + format = parser.get("format"); + maxsize = parser.get("max-size"); + + return true; + } + + virtual int Execute(const CaptureOptions &) + { + FileType type = FileType::JPG; + + if(format == "png") + { + type = FileType::PNG; + } + else if(format == "tga") + { + type = FileType::TGA; + } + else if(format == "bmp") + { + type = FileType::BMP; + } + else + { + const char *dot = strrchr(outfile.c_str(), '.'); + + if(dot != NULL && strstr(dot, "png")) + type = FileType::PNG; + else if(dot != NULL && strstr(dot, "tga")) + type = FileType::TGA; + else if(dot != NULL && strstr(dot, "bmp")) + type = FileType::BMP; + else if(dot != NULL && strstr(dot, "jpg")) + type = FileType::JPG; + else + std::cerr << "Couldn't guess format from '" << outfile << "', defaulting to jpg." + << std::endl; + } + + bytebuf buf; + + ICaptureFile *file = RENDERDOC_OpenCaptureFile(); + ResultDetails st = file->OpenFile(conv(infile), "rdc", NULL); + if(st.OK()) + { + buf = file->GetThumbnail(type, maxsize).data; + } + else + { + std::cerr << "Couldn't open '" << infile << "': " << st.Message() << std::endl; + } + file->Shutdown(); + + if(buf.empty()) + { + std::cerr << "Couldn't fetch the thumbnail in '" << infile << "'" << std::endl; + } + else + { + FILE *f = fopen(outfile.c_str(), "wb"); + + if(!f) + { + std::cerr << "Couldn't open destination file '" << outfile << "'" << std::endl; + } + else + { + fwrite(buf.data(), 1, buf.size(), f); + fclose(f); + + std::cout << "Wrote thumbnail from '" << infile << "' to '" << outfile << "'." << std::endl; + } + } + + return 0; + } +}; + struct RemoteServerCommand : public Command { private: @@ -1258,6 +1260,8 @@ public: } }; +#endif // !defined(RDOC_SELFCAPTURE_LIMITEDAPI) + struct VulkanRegisterCommand : public Command { private: @@ -1555,9 +1559,12 @@ int renderdoccmd(GlobalEnvironment &env, std::vector &argv) add_alias("/?", "help"); // add platform agnostic commands - add_command("thumb", new ThumbCommand()); + add_command("capture", new CaptureCommand()); add_command("inject", new InjectCommand()); + +#if !defined(RDOC_SELFCAPTURE_LIMITEDAPI) + add_command("thumb", new ThumbCommand()); add_command("remoteserver", new RemoteServerCommand()); add_command("replay", new ReplayCommand()); add_command("capaltbit", new CapAltBitCommand()); @@ -1565,6 +1572,7 @@ int renderdoccmd(GlobalEnvironment &env, std::vector &argv) add_command("convert", new ConvertCommand()); add_command("embed", new EmbeddedSectionCommand(false)); add_command("extract", new EmbeddedSectionCommand(true)); +#endif // !defined(RDOC_SELFCAPTURE_LIMITEDAPI) if(argv.size() <= 1) {